From 0ced6117e40cb69dca9e0676f3ff944d15e65720 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 10 May 2019 15:41:15 +0300 Subject: [PATCH 1/4] Run file verification in parallel threads Build librpm with OpenMP support, refactor the file loop to use the indexed rpmfiles objects instead of iterators. On my rusty old laptop, this tends to just slow things down unless things are hot in cache, in which case it roughly halves the running time. Might be worth more on SSD based setup. --- configure.ac | 1 + lib/Makefile.am | 1 + lib/verify.c | 41 +++++++++++++++++++++++++++-------------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/configure.ac b/configure.ac index b2d7ed8060..48edd3c144 100644 --- a/configure.ac +++ b/configure.ac @@ -12,6 +12,7 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES]) AC_CONFIG_TESTDIR(tests) AC_USE_SYSTEM_EXTENSIONS +AC_OPENMP AC_DISABLE_STATIC diff --git a/lib/Makefile.am b/lib/Makefile.am index 81f4eba61c..66840d02f4 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -2,6 +2,7 @@ include $(top_srcdir)/rpm.am AM_CFLAGS = @RPMCFLAGS@ +AM_CFLAGS += @OPENMP_CFLAGS@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir) -I$(top_builddir)/include/ AM_CPPFLAGS += @WITH_BEECRYPT_INCLUDE@ diff --git a/lib/verify.c b/lib/verify.c index d58454a002..323a08a2a3 100644 --- a/lib/verify.c +++ b/lib/verify.c @@ -344,19 +344,20 @@ static const char * stateStr(rpmfileState fstate) static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, rpmfileAttrs incAttrs, rpmfileAttrs skipAttrs) { - rpmVerifyAttrs verifyResult = 0; + rpmVerifyAttrs *results = NULL; rpmVerifyAttrs verifyAll = 0; /* assume no problems */ - rpmfi fi = rpmfiNew(ts, h, RPMTAG_BASENAMES, RPMFI_FLAGS_VERIFY); + rpmfiles fi = rpmfilesNew(NULL, h, RPMTAG_BASENAMES, RPMFI_FLAGS_VERIFY); + int i, fc = rpmfilesFC(fi); if (fi == NULL) return 1; - rpmfiInit(fi, 0); - while (rpmfiNext(fi) >= 0) { - rpmfileAttrs fileAttrs = rpmfiFFlags(fi); - char *buf = NULL, *attrFormat; - const char *fstate = NULL; - char ac; + results = xcalloc(fc, sizeof(*results)); + + #pragma omp parallel + #pragma omp single + for (i = 0; i < fc; i++) { + rpmfileAttrs fileAttrs = rpmfilesFFlags(fi, i); /* If filtering by inclusion, skip non-matching (eg --configfiles) */ if (incAttrs && !(incAttrs & fileAttrs)) @@ -366,12 +367,22 @@ static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, if (skipAttrs & fileAttrs) continue; - verifyResult = rpmfiVerify(fi, omitMask); + #pragma omp task + results[i] = rpmfilesVerify(fi, i, omitMask); + } + + for (i = 0; i < fc; i++) { + rpmfileAttrs fileAttrs = rpmfilesFFlags(fi, i); + rpmVerifyAttrs verifyResult = results[i]; + char *fn = rpmfilesFN(fi, i); + char *buf = NULL, *attrFormat; + const char *fstate = NULL; + char ac; /* Filter out timestamp differences of shared files */ if (verifyResult & RPMVERIFY_MTIME) { rpmdbMatchIterator mi; - mi = rpmtsInitIterator(ts, RPMDBI_BASENAMES, rpmfiFN(fi), 0); + mi = rpmtsInitIterator(ts, RPMDBI_BASENAMES, fn, 0); if (rpmdbGetIteratorCount(mi) > 1) verifyResult &= ~RPMVERIFY_MTIME; rpmdbFreeIterator(mi); @@ -379,13 +390,13 @@ static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, /* State is only meaningful for installed packages */ if (headerGetInstance(h)) - fstate = stateStr(rpmfiFState(fi)); + fstate = stateStr(rpmfilesFState(fi, i)); attrFormat = rpmFFlagsString(fileAttrs, ""); ac = rstreq(attrFormat, "") ? ' ' : attrFormat[0]; if (verifyResult & RPMVERIFY_LSTATFAIL) { if (!(fileAttrs & (RPMFILE_MISSINGOK|RPMFILE_GHOST)) || rpmIsVerbose()) { - rasprintf(&buf, _("missing %c %s"), ac, rpmfiFN(fi)); + rasprintf(&buf, _("missing %c %s"), ac, fn); if ((verifyResult & RPMVERIFY_LSTATFAIL) != 0 && errno != ENOENT) { char *app; @@ -396,7 +407,7 @@ static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, } } else if (verifyResult || fstate || rpmIsVerbose()) { char *verifyFormat = rpmVerifyString(verifyResult, "."); - rasprintf(&buf, "%s %c %s", verifyFormat, ac, rpmfiFN(fi)); + rasprintf(&buf, "%s %c %s", verifyFormat, ac, fn); free(verifyFormat); } free(attrFormat); @@ -413,8 +424,10 @@ static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, verifyResult &= ~RPMVERIFY_LSTATFAIL; verifyAll |= verifyResult; + free(fn); } - rpmfiFree(fi); + rpmfilesFree(fi); + free(results); return (verifyAll != 0) ? 1 : 0; } From b5da21861ad70b9df5c2cd5560a921251a5778d7 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 10 May 2019 16:17:17 +0300 Subject: [PATCH 2/4] fixup! Run file verification in parallel threads --- lib/verify.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/verify.c b/lib/verify.c index 323a08a2a3..93a0d168a5 100644 --- a/lib/verify.c +++ b/lib/verify.c @@ -332,6 +332,11 @@ static const char * stateStr(rpmfileState fstate) return _("unknown state"); } +struct filevfy_s { + rpmVerifyAttrs res; + int err; +}; + /** * Check file info from header against what's actually installed. * @param ts transaction set @@ -344,7 +349,7 @@ static const char * stateStr(rpmfileState fstate) static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, rpmfileAttrs incAttrs, rpmfileAttrs skipAttrs) { - rpmVerifyAttrs *results = NULL; + struct filevfy_s *results = NULL; rpmVerifyAttrs verifyAll = 0; /* assume no problems */ rpmfiles fi = rpmfilesNew(NULL, h, RPMTAG_BASENAMES, RPMFI_FLAGS_VERIFY); int i, fc = rpmfilesFC(fi); @@ -368,17 +373,29 @@ static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, continue; #pragma omp task - results[i] = rpmfilesVerify(fi, i, omitMask); + { + results[i].res = rpmfilesVerify(fi, i, omitMask); + results[i].err = errno; + } } for (i = 0; i < fc; i++) { rpmfileAttrs fileAttrs = rpmfilesFFlags(fi, i); - rpmVerifyAttrs verifyResult = results[i]; - char *fn = rpmfilesFN(fi, i); + rpmVerifyAttrs verifyResult = results[i].res; + char *fn = NULL; char *buf = NULL, *attrFormat; const char *fstate = NULL; char ac; + /* If filtering by inclusion, skip non-matching (eg --configfiles) */ + if (incAttrs && !(incAttrs & fileAttrs)) + continue; + + /* Skip on attributes (eg from --noghost) */ + if (skipAttrs & fileAttrs) + continue; + + fn = rpmfilesFN(fi, i); /* Filter out timestamp differences of shared files */ if (verifyResult & RPMVERIFY_MTIME) { rpmdbMatchIterator mi; @@ -398,9 +415,9 @@ static int verifyHeader(rpmts ts, Header h, rpmVerifyAttrs omitMask, if (!(fileAttrs & (RPMFILE_MISSINGOK|RPMFILE_GHOST)) || rpmIsVerbose()) { rasprintf(&buf, _("missing %c %s"), ac, fn); if ((verifyResult & RPMVERIFY_LSTATFAIL) != 0 && - errno != ENOENT) { + results[i].err != ENOENT) { char *app; - rasprintf(&app, " (%s)", strerror(errno)); + rasprintf(&app, " (%s)", strerror(results[i].err)); rstrcat(&buf, app); free(app); } From a084d16741e27e9bdbfcf0b8a020f46d11cd362e Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Mon, 13 May 2019 11:10:28 +0300 Subject: [PATCH 3/4] Run rpmkeys -K package signature/digest checking in parallel On my laptop this roughly doubles the runtime with cold cache, and halves it with a hot cache: checking packages on loopback-mounted Fedora-Server-dvd-x86_64-28-1.1.iso image takes about 33s serially and 60s in parallel when cold, and when hot, 20s serially and 10s parallel. --- lib/rpmchecksig.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/rpmchecksig.c b/lib/rpmchecksig.c index 810f7153d3..63bccca91e 100644 --- a/lib/rpmchecksig.c +++ b/lib/rpmchecksig.c @@ -288,7 +288,11 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv) rpmtsSetVfyLevel(ts, vfylevel); } + #pragma omp parallel reduction(+:res) + #pragma omp single while ((arg = *argv++) != NULL) { + #pragma omp task + { FD_t fd = Fopen(arg, "r.ufdio"); if (fd == NULL || Ferror(fd)) { rpmlog(RPMLOG_ERR, _("%s: open failed: %s\n"), @@ -299,6 +303,7 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv) } Fclose(fd); + } /* omp task */ rpmsqPoll(); } rpmKeyringFree(keyring); From 0814c18de263484e013cfe1a615fbe63c33c7082 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Mon, 13 May 2019 12:11:25 +0300 Subject: [PATCH 4/4] fixup! Run rpmkeys -K package signature/digest checking in parallel --- lib/rpmchecksig.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/rpmchecksig.c b/lib/rpmchecksig.c index 63bccca91e..0a28062f18 100644 --- a/lib/rpmchecksig.c +++ b/lib/rpmchecksig.c @@ -288,7 +288,7 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv) rpmtsSetVfyLevel(ts, vfylevel); } - #pragma omp parallel reduction(+:res) + #pragma omp parallel #pragma omp single while ((arg = *argv++) != NULL) { #pragma omp task @@ -297,8 +297,10 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv) if (fd == NULL || Ferror(fd)) { rpmlog(RPMLOG_ERR, _("%s: open failed: %s\n"), arg, Fstrerror(fd)); + #pragma omp atomic res++; } else if (rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, arg)) { + #pragma omp atomic res++; }