From f800d8e3412f0a18f888b27769e23ef3e1e4db41 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 11 Oct 2019 15:42:23 +0200 Subject: [PATCH 01/54] Implement fsync disabling for the ndb backend For some reason this never got implemented. Spotted by Jeff Johnson, thanks! (cherry picked from commit af64fe18ff03440502abc27b09d8db2b911b7d69) --- lib/backend/ndb/glue.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/backend/ndb/glue.c b/lib/backend/ndb/glue.c index 6d024f1beb..7778ed7745 100644 --- a/lib/backend/ndb/glue.c +++ b/lib/backend/ndb/glue.c @@ -30,6 +30,7 @@ struct ndbEnv_s { rpmpkgdb pkgdb; rpmxdb xdb; int refs; + int dofsync; unsigned int hdrNum; void *data; @@ -58,8 +59,10 @@ static void closeEnv(rpmdb rdb) static struct ndbEnv_s *openEnv(rpmdb rdb) { struct ndbEnv_s *ndbenv = rdb->db_dbenv; - if (!ndbenv) + if (!ndbenv) { rdb->db_dbenv = ndbenv = xcalloc(1, sizeof(struct ndbEnv_s)); + ndbenv->dofsync = 1; + } ndbenv->refs++; return ndbenv; } @@ -114,6 +117,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) } free(path); dbi->dbi_db = ndbenv->pkgdb = pkgdb; + rpmpkgSetFsync(pkgdb, ndbenv->dofsync); if ((oflags & (O_RDWR | O_RDONLY)) == O_RDONLY) dbi->dbi_flags |= DBI_RDONLY; @@ -148,6 +152,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) return 1; } free(path); + rpmxdbSetFsync(ndbenv->xdb, ndbenv->dofsync); } if (rpmxdbLookupBlob(ndbenv->xdb, &id, rpmtag, 0, 0) == RPMRC_NOTFOUND) { dbi->dbi_flags |= DBI_CREATED; @@ -179,6 +184,14 @@ static int ndb_Verify(dbiIndex dbi, unsigned int flags) static void ndb_SetFSync(rpmdb rdb, int enable) { + struct ndbEnv_s *ndbenv = rdb->db_dbenv; + if (ndbenv) { + ndbenv->dofsync = enable; + if (ndbenv->pkgdb) + rpmpkgSetFsync(ndbenv->pkgdb, enable); + if (ndbenv->xdb) + rpmxdbSetFsync(ndbenv->xdb, enable); + } } static int indexSync(rpmpkgdb pkgdb, rpmxdb xdb) From 8d5f880ee259e8635702cddd67881f1998125c00 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 11 Oct 2019 15:44:20 +0200 Subject: [PATCH 02/54] Delete all the IDXDB_FILESUPPORT code in the ndb backend Rpm will always use xdb instead of plain files for the database indexes. (cherry picked from commit 1c45d96b3218b4ecedb290f89d5230ed1acdd5ff) --- lib/backend/ndb/rpmidx.c | 290 ++++----------------------------------- 1 file changed, 30 insertions(+), 260 deletions(-) diff --git a/lib/backend/ndb/rpmidx.c b/lib/backend/ndb/rpmidx.c index 707c479347..25d2502f2d 100644 --- a/lib/backend/ndb/rpmidx.c +++ b/lib/backend/ndb/rpmidx.c @@ -66,8 +66,6 @@ typedef struct rpmidxdb_s { rpmpkgdb pkgdb; /* master database */ - char *filename; - int fd; /* our file descriptor */ int flags; int mode; @@ -166,36 +164,13 @@ static void mapcb(rpmxdb xdb, void *data, void *newaddr, size_t newsize) { set_mapped((rpmidxdb)data, newaddr, (unsigned int)newsize); } -static int rpmidxReadHeader(rpmidxdb idxdb); - static int rpmidxMap(rpmidxdb idxdb) { - if (idxdb->xdb) { - if (rpmxdbMapBlob(idxdb->xdb, idxdb->xdbid, idxdb->rdonly ? O_RDONLY : O_RDWR, mapcb, idxdb)) - return RPMRC_FAIL; - if (idxdb->file_size < 4096) { - rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); - return RPMRC_FAIL; - } - } else { -#ifdef IDXDB_FILESUPPORT - struct stat stb; - size_t size; - void *mapped; - if (fstat(idxdb->fd, &stb)) - return RPMRC_FAIL; - size = stb.st_size; - if (size < 4096) - return RPMRC_FAIL; - /* round up for mmap */ - size = (size + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - mapped = mmap(0, size, idxdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED, idxdb->fd, 0); - if (mapped == MAP_FAILED) - return RPMRC_FAIL; - set_mapped(idxdb, mapped, (unsigned int)stb.st_size); -#else + if (rpmxdbMapBlob(idxdb->xdb, idxdb->xdbid, idxdb->rdonly ? O_RDONLY : O_RDWR, mapcb, idxdb)) + return RPMRC_FAIL; + if (idxdb->file_size < 4096) { + rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); return RPMRC_FAIL; -#endif } return RPMRC_OK; } @@ -204,50 +179,9 @@ static void rpmidxUnmap(rpmidxdb idxdb) { if (!idxdb->head_mapped) return; - if (idxdb->xdb) { - rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); - } else { -#ifdef IDXDB_FILESUPPORT - size_t size = idxdb->file_size; - /* round up for munmap */ - size = (size + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - munmap(idxdb->head_mapped, size); - set_mapped(idxdb, 0, 0); -#else - return; -#endif - } + rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); } -#ifdef IDXDB_FILESUPPORT -static int rpmidxReadHeader(rpmidxdb idxdb); - -/* re-open file to get the new version */ -static int rpmidxHandleObsolete(rpmidxdb idxdb) -{ - int nfd; - struct stat stb1, stb2; - - if (fstat(idxdb->fd, &stb1)) - return RPMRC_FAIL; - nfd = open(idxdb->filename, idxdb->rdonly ? O_RDONLY : O_RDWR, 0); - if (nfd == -1) - return RPMRC_FAIL; - if (fstat(nfd, &stb2)) { - close(nfd); - return RPMRC_FAIL; - } - if (stb1.st_dev == stb2.st_dev && stb1.st_ino == stb2.st_ino) { - close(nfd); - return RPMRC_FAIL; /* opened the same obsolete file */ - } - rpmidxUnmap(idxdb); - close(idxdb->fd); - idxdb->fd = nfd; - return rpmidxReadHeader(idxdb); /* re-try with new file */ -} -#endif - static int rpmidxReadHeader(rpmidxdb idxdb) { unsigned int version; @@ -272,10 +206,6 @@ static int rpmidxReadHeader(rpmidxdb idxdb) rpmidxUnmap(idxdb); return RPMRC_FAIL; } -#ifdef IDXDB_FILESUPPORT - if (!idxdb->xdb && le2ha(idxdb->head_mapped + IDXDB_OFFSET_OBSOLETE)) - return rpmidxHandleObsolete(idxdb); -#endif idxdb->generation = le2ha(idxdb->head_mapped + IDXDB_OFFSET_GENERATION); idxdb->nslots = le2ha(idxdb->head_mapped + IDXDB_OFFSET_NSLOTS); idxdb->usedslots = le2ha(idxdb->head_mapped + IDXDB_OFFSET_USEDSLOTS); @@ -338,23 +268,6 @@ static inline void bumpGeneration(rpmidxdb idxdb) h2lea(idxdb->generation, idxdb->head_mapped + IDXDB_OFFSET_GENERATION); } -#ifdef IDXDB_FILESUPPORT -static int createempty(rpmidxdb idxdb, off_t off, size_t size) -{ - char buf[4096]; - memset(buf, 0, sizeof(buf)); - while (size >= 4096) { - if (pwrite(idxdb->fd, buf, 4096, off) != 4096) - return RPMRC_FAIL; - off += 4096; - size -= 4096; - } - if (size > 0 && pwrite(idxdb->fd, buf, size , off) != size) - return RPMRC_FAIL; - return RPMRC_OK; -} -#endif - /*** Key management ***/ #define MURMUR_M 0x5bd1e995 @@ -454,29 +367,8 @@ static inline int equalkey(rpmidxdb idxdb, unsigned int off, const unsigned char static int addkeypage(rpmidxdb idxdb) { unsigned int addsize = idxdb->pagesize > IDXDB_KEY_CHUNKSIZE ? idxdb->pagesize : IDXDB_KEY_CHUNKSIZE; - if (idxdb->xdb) { - if (rpmxdbResizeBlob(idxdb->xdb, idxdb->xdbid, idxdb->file_size + addsize)) - return RPMRC_FAIL; - } else { -#ifdef IDXDB_FILESUPPORT - /* don't use ftruncate because we want to create a "backed" page */ - void *newaddr; - size_t oldsize, newsize; - if (createempty(idxdb, idxdb->file_size, addsize)) - return RPMRC_FAIL; - oldsize = idxdb->file_size; - newsize = idxdb->file_size + addsize; - /* round up for mremap */ - oldsize = (oldsize + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - newsize = (newsize + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - newaddr = mremap(idxdb->head_mapped, oldsize, newsize, MREMAP_MAYMOVE); - if (newaddr == MAP_FAILED) - return RPMRC_FAIL; - set_mapped(idxdb, newaddr, idxdb->file_size + addsize); -#else + if (rpmxdbResizeBlob(idxdb->xdb, idxdb->xdbid, idxdb->file_size + addsize)) return RPMRC_FAIL; -#endif - } return RPMRC_OK; } @@ -629,45 +521,16 @@ static int rpmidxRebuildInternal(rpmidxdb idxdb) nidxdb->xmask = xmask; /* create new database */ - if (idxdb->xdb) { - nidxdb->xdb = idxdb->xdb; - nidxdb->xdbtag = idxdb->xdbtag; - if (rpmxdbLookupBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG_REBUILD, O_CREAT|O_TRUNC)) { - return RPMRC_FAIL; - } - if (rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, file_size)) { - return RPMRC_FAIL; - } - if (rpmidxMap(nidxdb)) { - return RPMRC_FAIL; - } - } else { -#ifdef IDXDB_FILESUPPORT - void *mapped; - nidxdb->filename = xmalloc(strlen(idxdb->filename) + 8); - sprintf(nidxdb->filename, "%s-XXXXXX", idxdb->filename); - nidxdb->fd = mkstemp(nidxdb->filename); - if (nidxdb->fd == -1) { - free(nidxdb->filename); - return RPMRC_FAIL; - } - if (createempty(nidxdb, 0, file_size)) { - close(nidxdb->fd); - unlink(nidxdb->filename); - free(nidxdb->filename); - return RPMRC_FAIL; - } - mapped = mmap(0, file_size, idxdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED, nidxdb->fd, 0); - if (mapped == MAP_FAILED) { - close(nidxdb->fd); - unlink(nidxdb->filename); - free(nidxdb->filename); - return RPMRC_FAIL; - } - set_mapped(nidxdb, mapped, file_size); -#else + nidxdb->xdb = idxdb->xdb; + nidxdb->xdbtag = idxdb->xdbtag; + if (rpmxdbLookupBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG_REBUILD, O_CREAT|O_TRUNC)) { + return RPMRC_FAIL; + } + if (rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, file_size)) { + return RPMRC_FAIL; + } + if (rpmidxMap(nidxdb)) { return RPMRC_FAIL; -#endif } /* copy all entries */ @@ -700,42 +563,14 @@ static int rpmidxRebuildInternal(rpmidxdb idxdb) /* shrink if we have allocated excessive key space */ xfile_size = file_size - key_size + keyend + IDXDB_KEY_CHUNKSIZE; xfile_size = (xfile_size + nidxdb->pagesize - 1) & ~(nidxdb->pagesize - 1); - if (xfile_size < file_size) { - if (nidxdb->xdb) { - rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, xfile_size); - } else { - if (ftruncate(nidxdb->fd, xfile_size)) { - rpmlog(RPMLOG_WARNING, _("truncate failed: %s\n"), strerror(errno)); - } - } - } + if (xfile_size < file_size) + rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, xfile_size); /* now switch over to new database */ - if (idxdb->xdb) { - rpmidxUnmap(idxdb); - if (rpmxdbRenameBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG)) - return RPMRC_FAIL; - idxdb->xdbid = nidxdb->xdbid; - } else { -#ifdef IDXDB_FILESUPPORT - if (rename(nidxdb->filename, idxdb->filename)) { - close(nidxdb->fd); - unlink(nidxdb->filename); - free(nidxdb->filename); - return RPMRC_FAIL; - } - if (idxdb->head_mapped) { - h2lea(1, idxdb->head_mapped + IDXDB_OFFSET_OBSOLETE); - bumpGeneration(idxdb); - rpmidxUnmap(idxdb); - } - free(nidxdb->filename); - close(idxdb->fd); - idxdb->fd = nidxdb->fd; -#else + rpmidxUnmap(idxdb); + if (rpmxdbRenameBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG)) return RPMRC_FAIL; -#endif - } + idxdb->xdbid = nidxdb->xdbid; if (rpmidxReadHeader(idxdb)) return RPMRC_FAIL; return RPMRC_OK; @@ -1001,26 +836,14 @@ static int rpmidxListInternal(rpmidxdb idxdb, unsigned int **keylistp, unsigned static int rpmidxInitInternal(rpmidxdb idxdb) { - if (idxdb->xdb) { - unsigned int id; - int rc = rpmxdbLookupBlob(idxdb->xdb, &id, idxdb->xdbtag, IDXDB_XDB_SUBTAG, 0); - if (rc == RPMRC_OK && id) { - idxdb->xdbid = id; - return RPMRC_OK; /* somebody else was faster */ - } - if (rc && rc != RPMRC_NOTFOUND) - return rc; - } else { -#ifdef IDXDB_FILESUPPORT - struct stat stb; - if (stat(idxdb->filename, &stb)) - return RPMRC_FAIL; - if (stb.st_size) /* somebody else was faster */ - return rpmidxHandleObsolete(idxdb); -#else - return RPMRC_FAIL; -#endif + unsigned int id; + int rc = rpmxdbLookupBlob(idxdb->xdb, &id, idxdb->xdbtag, IDXDB_XDB_SUBTAG, 0); + if (rc == RPMRC_OK && id) { + idxdb->xdbid = id; + return RPMRC_OK; /* somebody else was faster */ } + if (rc && rc != RPMRC_NOTFOUND) + return rc; return rpmidxRebuildInternal(idxdb); } @@ -1028,18 +851,12 @@ static int rpmidxLock(rpmidxdb idxdb, int excl) { if (excl && idxdb->rdonly) return RPMRC_FAIL; - if (idxdb->xdb) - return rpmxdbLock(idxdb->xdb, excl); - else - return rpmpkgLock(idxdb->pkgdb, excl); + return rpmxdbLock(idxdb->xdb, excl); } static int rpmidxUnlock(rpmidxdb idxdb, int excl) { - if (idxdb->xdb) - return rpmxdbUnlock(idxdb->xdb, excl); - else - return rpmpkgUnlock(idxdb->pkgdb, excl); + return rpmxdbUnlock(idxdb->xdb, excl); } static int rpmidxLockReadHeader(rpmidxdb idxdb, int excl) @@ -1065,43 +882,7 @@ static int rpmidxInit(rpmidxdb idxdb) int rpmidxOpen(rpmidxdb *idxdbp, rpmpkgdb pkgdb, const char *filename, int flags, int mode) { -#ifdef IDXDB_FILESUPPORT - struct stat stb; - rpmidxdb idxdb; - - *idxdbp = 0; - idxdb = xcalloc(1, sizeof(*idxdb)); - idxdb->filename = xstrdup(filename); - if ((flags & (O_RDONLY|O_RDWR)) == O_RDONLY) - idxdb->rdonly = 1; - if ((idxdb->fd = open(filename, flags, mode)) == -1) { - free(idxdb->filename); - free(idxdb); - return RPMRC_FAIL; - } - if (fstat(idxdb->fd, &stb)) { - close(idxdb->fd); - free(idxdb->filename); - free(idxdb); - return RPMRC_FAIL; - } - idxdb->pkgdb = pkgdb; - idxdb->flags = flags; - idxdb->mode = mode; - idxdb->pagesize = sysconf(_SC_PAGE_SIZE); - if (stb.st_size == 0) { - if (rpmidxInit(idxdb)) { - close(idxdb->fd); - free(idxdb->filename); - free(idxdb); - return RPMRC_FAIL; - } - } - *idxdbp = idxdb; - return RPMRC_OK; -#else return RPMRC_FAIL; -#endif } int rpmidxOpenXdb(rpmidxdb *idxdbp, rpmpkgdb pkgdb, rpmxdb xdb, unsigned int xdbtag) @@ -1121,7 +902,6 @@ int rpmidxOpenXdb(rpmidxdb *idxdbp, rpmpkgdb pkgdb, rpmxdb xdb, unsigned int xdb return RPMRC_FAIL; } idxdb = xcalloc(1, sizeof(*idxdb)); - idxdb->fd = -1; idxdb->xdb = xdb; idxdb->xdbtag = xdbtag; idxdb->xdbid = id; @@ -1165,12 +945,6 @@ int rpmidxDelXdb(rpmpkgdb pkgdb, rpmxdb xdb, unsigned int xdbtag) void rpmidxClose(rpmidxdb idxdb) { rpmidxUnmap(idxdb); - if (idxdb->fd >= 0) { - close(idxdb->fd); - idxdb->fd = -1; - } - if (idxdb->filename) - free(idxdb->filename); free(idxdb); } @@ -1229,11 +1003,7 @@ int rpmidxStats(rpmidxdb idxdb) if (rpmidxLockReadHeader(idxdb, 0)) return RPMRC_FAIL; printf("--- IndexDB Stats\n"); - if (idxdb->xdb) { - printf("Xdb tag: %d, id: %d\n", idxdb->xdbtag, idxdb->xdbid); - } else { - printf("Filename: %s\n", idxdb->filename); - } + printf("Xdb tag: %d, id: %d\n", idxdb->xdbtag, idxdb->xdbid); printf("Generation: %u\n", idxdb->generation); printf("Slots: %u\n", idxdb->nslots); printf("Used slots: %u\n", idxdb->usedslots); From b0cf860b3ead90170edd4ac15db9c9867d6d8f5b Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 11 Oct 2019 15:48:41 +0200 Subject: [PATCH 03/54] Do not always fsync the database directories Only call fsync when new database files got created. (cherry picked from commit 09cc8ebad29aeaca58d009fd9d1a789919282cc7) --- lib/backend/ndb/rpmpkg.c | 50 +++++++++++++++++++--------------------- lib/backend/ndb/rpmxdb.c | 49 +++++++++++++++++++-------------------- 2 files changed, 48 insertions(+), 51 deletions(-) diff --git a/lib/backend/ndb/rpmpkg.c b/lib/backend/ndb/rpmpkg.c index 80a4b303bb..fe1501352c 100644 --- a/lib/backend/ndb/rpmpkg.c +++ b/lib/backend/ndb/rpmpkg.c @@ -840,6 +840,23 @@ static int rpmpkgInit(rpmpkgdb pkgdb) return rc; } +static int rpmpkgFsyncDir(const char *filename) +{ + int rc = RPMRC_OK; + DIR *pdir; + char *filenameCopy = xstrdup(filename); + + if ((pdir = opendir(dirname(filenameCopy))) == NULL) { + free(filenameCopy); + return RPMRC_FAIL; + } + if (fsync(dirfd(pdir)) == -1) + rc = RPMRC_FAIL; + closedir(pdir); + free(filenameCopy); + return rc; +} + int rpmpkgOpen(rpmpkgdb *pkgdbp, const char *filename, int flags, int mode) { struct stat stb; @@ -855,32 +872,6 @@ int rpmpkgOpen(rpmpkgdb *pkgdbp, const char *filename, int flags, int mode) free(pkgdb); return RPMRC_FAIL; } - if (flags & O_CREAT) { - char *filenameCopy; - DIR *pdir; - - filenameCopy = xstrdup(pkgdb->filename); - - if ((pdir = opendir(dirname(filenameCopy))) == NULL) { - free(filenameCopy); - close(pkgdb->fd); - free(pkgdb->filename); - free(pkgdb); - return RPMRC_FAIL; - } - - if (fsync(dirfd(pdir)) == -1) { - closedir(pdir); - free(filenameCopy); - close(pkgdb->fd); - free(pkgdb->filename); - free(pkgdb); - return RPMRC_FAIL; - } - closedir(pdir); - free(filenameCopy); - - } if (fstat(pkgdb->fd, &stb)) { close(pkgdb->fd); free(pkgdb->filename); @@ -888,6 +879,13 @@ int rpmpkgOpen(rpmpkgdb *pkgdbp, const char *filename, int flags, int mode) return RPMRC_FAIL; } if (stb.st_size == 0) { + /* created new database */ + if (rpmpkgFsyncDir(pkgdb->filename)) { + close(pkgdb->fd); + free(pkgdb->filename); + free(pkgdb); + return RPMRC_FAIL; + } if (rpmpkgInit(pkgdb)) { close(pkgdb->fd); free(pkgdb->filename); diff --git a/lib/backend/ndb/rpmxdb.c b/lib/backend/ndb/rpmxdb.c index 0ff56aa023..f136330a8e 100644 --- a/lib/backend/ndb/rpmxdb.c +++ b/lib/backend/ndb/rpmxdb.c @@ -480,6 +480,23 @@ static int rpmxdbInit(rpmxdb xdb) return rc; } +static int rpmxdbFsyncDir(const char *filename) +{ + int rc = RPMRC_OK; + DIR *pdir; + char *filenameCopy = xstrdup(filename); + + if ((pdir = opendir(dirname(filenameCopy))) == NULL) { + free(filenameCopy); + return RPMRC_FAIL; + } + if (fsync(dirfd(pdir)) == -1) + rc = RPMRC_FAIL; + closedir(pdir); + free(filenameCopy); + return rc; +} + int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pkgdb, const char *filename, int flags, int mode) { struct stat stb; @@ -497,31 +514,6 @@ int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pkgdb, const char *filename, int flags, in free(xdb); return RPMRC_FAIL; } - if (flags & O_CREAT) { - char *filenameCopy; - DIR *pdir; - - filenameCopy = xstrdup(xdb->filename); - - if ((pdir = opendir(dirname(filenameCopy))) == NULL) { - free(filenameCopy); - close(xdb->fd); - free(xdb->filename); - free(xdb); - return RPMRC_FAIL; - } - - if (fsync(dirfd(pdir)) == -1) { - closedir(pdir); - free(filenameCopy); - close(xdb->fd); - free(xdb->filename); - free(xdb); - return RPMRC_FAIL; - } - closedir(pdir); - free(filenameCopy); - } if (fstat(xdb->fd, &stb)) { close(xdb->fd); free(xdb->filename); @@ -529,6 +521,13 @@ int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pkgdb, const char *filename, int flags, in return RPMRC_FAIL; } if (stb.st_size == 0) { + /* created new database */ + if (rpmxdbFsyncDir(xdb->filename)) { + close(xdb->fd); + free(xdb->filename); + free(xdb); + return RPMRC_FAIL; + } if (rpmxdbInit(xdb)) { close(xdb->fd); free(xdb->filename); From aa69937a3fb19cce89e2b9c3e961dbd87e3befc2 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 11 Oct 2019 15:55:43 +0200 Subject: [PATCH 04/54] Use fdatasync in ndb if available No need to always sync the inode metadata. (cherry picked from commit 9ac6c427d240dca7375dfff596f8a84012c32169) --- lib/backend/ndb/rpmpkg.c | 17 +++++++++++++---- lib/backend/ndb/rpmxdb.c | 11 ++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/backend/ndb/rpmpkg.c b/lib/backend/ndb/rpmpkg.c index fe1501352c..0809af19e2 100644 --- a/lib/backend/ndb/rpmpkg.c +++ b/lib/backend/ndb/rpmpkg.c @@ -156,6 +156,15 @@ static int rpmpkgReadHeader(rpmpkgdb pkgdb) return RPMRC_OK; } +static int rpmpkgFsync(rpmpkgdb pkgdb) +{ +#ifdef HAVE_FDATASYNC + return fdatasync(pkgdb->fd); +#else + return fsync(pkgdb->fd); +#endif +} + static int rpmpkgWriteHeader(rpmpkgdb pkgdb) { unsigned char header[PKGDB_HEADER_SIZE]; @@ -168,7 +177,7 @@ static int rpmpkgWriteHeader(rpmpkgdb pkgdb) if (pwrite(pkgdb->fd, header, sizeof(header), 0) != sizeof(header)) { return RPMRC_FAIL; } - if (pkgdb->dofsync && fsync(pkgdb->fd)) + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) return RPMRC_FAIL; /* write error */ return RPMRC_OK; } @@ -440,7 +449,7 @@ static int rpmpkgWriteEmptySlotpage(rpmpkgdb pkgdb, int pageno) if (pwrite(pkgdb->fd, page, PAGE_SIZE - off, pageno * PAGE_SIZE + off) != PAGE_SIZE - off) { return RPMRC_FAIL; } - if (pkgdb->dofsync && fsync(pkgdb->fd)) { + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) { return RPMRC_FAIL; /* write error */ } return RPMRC_OK; @@ -651,7 +660,7 @@ static int rpmpkgWriteBlob(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned int blk /* update file length */ if (blkoff + blkcnt > pkgdb->fileblks) pkgdb->fileblks = blkoff + blkcnt; - if (pkgdb->dofsync && fsync(pkgdb->fd)) { + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) { return RPMRC_FAIL; /* write error */ } return RPMRC_OK; @@ -663,7 +672,7 @@ static int rpmpkgDelBlob(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned int blkof return RPMRC_FAIL; if (rpmpkgZeroBlks(pkgdb, blkoff, blkcnt)) return RPMRC_FAIL; - if (pkgdb->dofsync && fsync(pkgdb->fd)) + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) return RPMRC_FAIL; /* write error */ return RPMRC_OK; } diff --git a/lib/backend/ndb/rpmxdb.c b/lib/backend/ndb/rpmxdb.c index f136330a8e..97446728df 100644 --- a/lib/backend/ndb/rpmxdb.c +++ b/lib/backend/ndb/rpmxdb.c @@ -1127,12 +1127,21 @@ int rpmxdbIsRdonly(rpmxdb xdb) return xdb->rdonly; } +static int rpmxdbFsync(rpmxdb xdb) +{ +#ifdef HAVE_FDATASYNC + return fdatasync(xdb->fd); +#else + return fsync(xdb->fd); +#endif +} + int rpmxdbSetUserGeneration(rpmxdb xdb, unsigned int usergeneration) { if (rpmxdbLockReadHeader(xdb, 1)) return RPMRC_FAIL; /* sync before the update */ - if (xdb->dofsync && fsync(xdb->fd)) { + if (xdb->dofsync && rpmxdbFsync(xdb)) { rpmxdbUnlock(xdb, 1); return RPMRC_FAIL; } From ef3ebd0bd18883728264575f37c592e0e37a6762 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 11 Oct 2019 15:57:47 +0200 Subject: [PATCH 05/54] Refactor mmap/munmap/mremap handling in ndb Also emulate mremap with mmap/munmap in ndb if it is not available. (cherry picked from commit 3625ca14c752fa229c79891fcc6374df40b5b588) --- configure.ac | 6 +-- lib/backend/ndb/rpmxdb.c | 106 ++++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 45 deletions(-) diff --git a/configure.ac b/configure.ac index 1165a4c865..62703ee135 100644 --- a/configure.ac +++ b/configure.ac @@ -561,10 +561,8 @@ yes|no) ;; esac], [enable_ndb=no]) AS_IF([test "$enable_ndb" = yes],[ - AC_CHECK_FUNCS([mremap], - [AC_DEFINE(ENABLE_NDB, 1, [Enable new rpm database format?])], - [AC_MSG_ERROR([mremap function required by ndb])], - [#include ]) + AC_DEFINE(ENABLE_NDB, 1, [Enable new rpm database format?]) + AC_CHECK_FUNCS(mremap, [], [], [#include ]) ]) AM_CONDITIONAL([NDB], [test "$enable_ndb" = yes]) diff --git a/lib/backend/ndb/rpmxdb.c b/lib/backend/ndb/rpmxdb.c index 97446728df..f7c6ebd91b 100644 --- a/lib/backend/ndb/rpmxdb.c +++ b/lib/backend/ndb/rpmxdb.c @@ -36,6 +36,7 @@ typedef struct rpmxdb_s { unsigned int usergeneration; unsigned char *mapped; + int mapflags; unsigned int mappedlen; struct xdb_slot { @@ -95,12 +96,52 @@ static inline void h2lea(unsigned int x, unsigned char *p) #define SLOT_MAGIC ('S' | 'l' << 8 | 'o' << 16) -#define SLOT_SIZE 16 +#define SLOT_SIZE 16 #define SLOT_START (XDB_HEADER_SIZE / SLOT_SIZE) -static void rpmxdbUnmap(rpmxdb xdb) + +/* low level map/remap a file into memory */ +static void *mapmem(void *oldaddr, size_t oldsize, size_t newsize, int prot, int fd, off_t offset) +{ + if (oldaddr) { +#if HAVE_MREMAP + return mremap(oldaddr, oldsize, newsize, MREMAP_MAYMOVE); +#else + void *mapped = mmap(0, newsize, prot, MAP_SHARED, fd, offset); + if (mapped != MAP_FAILED) + munmap(oldaddr, oldsize); + return mapped; +#endif + } + return mmap(0, newsize, prot, MAP_SHARED, fd, offset); +} + +/* unmap a mapped region */ +static void unmapmem(void *addr, size_t size) +{ + munmap(addr, size); +} + +#define ROUNDTOSYSTEMPAGE(xdb, size) (((size) + (xdb->systempagesize - 1)) & ~(xdb->systempagesize - 1)) + +/* xdb header mapping functions */ +static int mapheader(rpmxdb xdb, unsigned int slotnpages) +{ + unsigned char *mapped; + size_t mappedlen = slotnpages * xdb->pagesize; + + mappedlen = ROUNDTOSYSTEMPAGE(xdb, mappedlen); + mapped = mapmem(xdb->mapped, xdb->mappedlen, mappedlen, xdb->mapflags, xdb->fd, 0); + if ((void *)mapped == MAP_FAILED) + return RPMRC_FAIL; + xdb->mapped = mapped; + xdb->mappedlen = mappedlen; + return RPMRC_OK; +} + +static void unmapheader(rpmxdb xdb) { - munmap(xdb->mapped, xdb->mappedlen); + unmapmem(xdb->mapped, xdb->mappedlen); xdb->mapped = 0; xdb->mappedlen = 0; } @@ -120,9 +161,9 @@ static int mapslot(rpmxdb xdb, struct xdb_slot *slot) shift = off & (xdb->systempagesize - 1); off -= shift; size += shift; - size = (size + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); + size = ROUNDTOSYSTEMPAGE(xdb, size); } - mapped = mmap(0, size, slot->mapflags, MAP_SHARED, xdb->fd, off); + mapped = mapmem(0, 0, size, slot->mapflags, xdb->fd, off); if (mapped == MAP_FAILED) return RPMRC_FAIL; slot->mapped = (unsigned char *)mapped + shift; @@ -131,44 +172,41 @@ static int mapslot(rpmxdb xdb, struct xdb_slot *slot) static void unmapslot(rpmxdb xdb, struct xdb_slot *slot) { - size_t size; unsigned char *mapped = slot->mapped; + size_t size; if (!mapped) return; size = slot->pagecnt * xdb->pagesize; if (xdb->pagesize != xdb->systempagesize) { size_t off = slot->startpage * xdb->pagesize; size_t shift = off & (xdb->systempagesize - 1); - mapped -= shift; size += shift; - size = (size + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); + size = ROUNDTOSYSTEMPAGE(xdb, size); + mapped -= shift; } - munmap(mapped, size); + unmapmem(mapped, size); slot->mapped = 0; } static int remapslot(rpmxdb xdb, struct xdb_slot *slot, unsigned int newpagecnt) { - void *mapped; + unsigned char *mapped = slot->mapped; size_t off, oldsize, newsize, shift; + oldsize = slot->pagecnt * xdb->pagesize; newsize = newpagecnt * xdb->pagesize; off = slot->startpage * xdb->pagesize; shift = 0; if (xdb->pagesize != xdb->systempagesize) { - off = slot->startpage * xdb->pagesize; shift = off & (xdb->systempagesize - 1); off -= shift; oldsize += shift; - oldsize = (oldsize + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); newsize += shift; - newsize = (newsize + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); + oldsize = ROUNDTOSYSTEMPAGE(xdb, oldsize); + newsize = ROUNDTOSYSTEMPAGE(xdb, newsize); } - if (slot->mapped) - mapped = mremap(slot->mapped - shift, oldsize, newsize, MREMAP_MAYMOVE); - else - mapped = mmap(0, newsize, slot->mapflags, MAP_SHARED, xdb->fd, off); - if (mapped == MAP_FAILED) + mapped = mapmem(mapped ? mapped - shift : 0, oldsize, newsize, slot->mapflags, xdb->fd, off); + if ((void *)mapped == MAP_FAILED) return RPMRC_FAIL; slot->mapped = (unsigned char *)mapped + shift; slot->pagecnt = newpagecnt; @@ -198,13 +236,12 @@ static int rpmxdbReadHeader(rpmxdb xdb) unsigned int usedblobpages; int i, nused, slotno; struct stat stb; - size_t mapsize; if (xdb->mapped) { if (le2ha(xdb->mapped + XDB_OFFSET_GENERATION) == xdb->generation) { return RPMRC_OK; } - rpmxdbUnmap(xdb); + unmapheader(xdb); } if (fstat(xdb->fd, &stb)) { return RPMRC_FAIL; @@ -228,16 +265,10 @@ static int rpmxdbReadHeader(rpmxdb xdb) if (!slotnpages || !pagesize || stb.st_size % pagesize != 0) return RPMRC_FAIL; xdb->pagesize = pagesize; + xdb->mapflags = xdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE; - /* round up */ - mapsize = slotnpages * pagesize; - mapsize = (mapsize + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); - xdb->mapped = mmap(0, mapsize, xdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED, xdb->fd, 0); - if ((void *)xdb->mapped == MAP_FAILED) { - xdb->mapped = 0; + if (mapheader(xdb, slotnpages)) return RPMRC_FAIL; - } - xdb->mappedlen = mapsize; /* read in all slots */ xdb->firstfree = 0; @@ -258,7 +289,7 @@ static int rpmxdbReadHeader(rpmxdb xdb) if ((slot->subtag & 0x00ffffff) != SLOT_MAGIC) { free(slots); free(usedslots); - rpmxdbUnmap(xdb); + unmapheader(xdb); return RPMRC_FAIL; } slot->subtag = (slot->subtag >> 24) & 255; @@ -287,7 +318,7 @@ static int rpmxdbReadHeader(rpmxdb xdb) if (lastslot->startpage + lastslot->pagecnt > slot->startpage) { free(slots); free(usedslots); - rpmxdbUnmap(xdb); + unmapheader(xdb); return RPMRC_FAIL; } lastslot->next = slot->slotno; @@ -705,10 +736,8 @@ static int moveblobstofront(rpmxdb xdb, struct xdb_slot *afterslot) /* add a single page containing empty slots */ static int addslotpage(rpmxdb xdb) { - unsigned char *newaddr; struct xdb_slot *slot; int i, spp, nslots; - size_t newmappedlen; if (xdb->firstfree) return RPMRC_FAIL; @@ -730,17 +759,12 @@ static int addslotpage(rpmxdb xdb) slot = xrealloc(xdb->slots, (nslots + 1 + spp) * sizeof(*slot)); xdb->slots = slot; - if (rpmxdbWriteEmptySlotpage(xdb, xdb->slotnpages)) { + if (rpmxdbWriteEmptySlotpage(xdb, xdb->slotnpages)) return RPMRC_FAIL; - } - /* remap slots */ - newmappedlen = xdb->slotnpages * xdb->pagesize + xdb->pagesize; - newmappedlen = (newmappedlen + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); - newaddr = mremap(xdb->mapped, xdb->mappedlen, newmappedlen, MREMAP_MAYMOVE); - if (newaddr == MAP_FAILED) + + /* remap the header */ + if (mapheader(xdb, xdb->slotnpages + 1)) return RPMRC_FAIL; - xdb->mapped = newaddr; - xdb->mappedlen = newmappedlen; /* update the header */ xdb->slotnpages++; From 47593bbe4a0ac6a6b30b68fc4c0b21909912d822 Mon Sep 17 00:00:00 2001 From: Igor Kanyuka Date: Wed, 16 Oct 2019 13:19:48 -0700 Subject: [PATCH 06/54] Increase lmdb DB size from 256M to 1G (cherry picked from commit a623c45ac7a566ee9c33325ef7a318e5ef7d248a) --- lib/backend/lmdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backend/lmdb.c b/lib/backend/lmdb.c index db1270ebce..26dadfb554 100644 --- a/lib/backend/lmdb.c +++ b/lib/backend/lmdb.c @@ -145,7 +145,7 @@ static int db_init(rpmdb rdb, const char * dbhome) MDB_dbi maxdbs = 32; unsigned int maxreaders = 16; - size_t mapsize = 256 * 1024 * 1024; + size_t mapsize = 1024 * 1024 * 1024; if ((rc = mdb_env_create(&env)) || (rc = mdb_env_set_maxreaders(env, maxreaders)) From 5d8472583509444d7f206411b6c874edf25c52f2 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Thu, 17 Oct 2019 12:43:20 +0300 Subject: [PATCH 07/54] Don't report unimplemented db ctrl and verify ops as errors ndb doesn't implement a specific verify option, but that doesn't mean the data within should be considered invalid. This causes ndb to fail the test-suite on "rpmdb --rebuilddb and verify empty database" for no good reason. Similar arguments could be made for dummydb although it matters much less there. (cherry picked from commit 7ba4b9bdcfc84e7e4740ede0c834581d3be1d865) --- lib/backend/dummydb.c | 4 ++-- lib/backend/ndb/glue.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/backend/dummydb.c b/lib/backend/dummydb.c index 961d641110..dbd0d9b448 100644 --- a/lib/backend/dummydb.c +++ b/lib/backend/dummydb.c @@ -22,7 +22,7 @@ static int dummydb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int fla static int dummydb_Verify(dbiIndex dbi, unsigned int flags) { - return 1; + return 0; } static void dummydb_SetFSync(rpmdb rdb, int enable) @@ -31,7 +31,7 @@ static void dummydb_SetFSync(rpmdb rdb, int enable) static int dummydb_Ctrl(rpmdb rdb, dbCtrlOp ctrl) { - return 1; + return 0; } static dbiCursor dummydb_CursorInit(dbiIndex dbi, unsigned int flags) diff --git a/lib/backend/ndb/glue.c b/lib/backend/ndb/glue.c index 7778ed7745..36c214dc68 100644 --- a/lib/backend/ndb/glue.c +++ b/lib/backend/ndb/glue.c @@ -179,7 +179,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) static int ndb_Verify(dbiIndex dbi, unsigned int flags) { - return 1; + return 0; } static void ndb_SetFSync(rpmdb rdb, int enable) From f84b3c7e9bea43df3f9f85ccdcc36172bcc42239 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 18 Oct 2019 14:27:42 +0200 Subject: [PATCH 08/54] Use xdb's pagesize instead of sysconf(_SC_PAGE_SIZE) Our xdb may use a different page size for some reason. (cherry picked from commit 20fd80802098fe91e85dbe37d1857f6af1e91193) --- lib/backend/ndb/rpmidx.c | 7 +++---- lib/backend/ndb/rpmxdb.c | 5 +++++ lib/backend/ndb/rpmxdb.h | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/backend/ndb/rpmidx.c b/lib/backend/ndb/rpmidx.c index 25d2502f2d..0ec8c8d119 100644 --- a/lib/backend/ndb/rpmidx.c +++ b/lib/backend/ndb/rpmidx.c @@ -483,7 +483,7 @@ static int rpmidxRebuildInternal(rpmidxdb idxdb) nidxdb = &nidxdb_s; memset(nidxdb, 0, sizeof(*nidxdb)); - nidxdb->pagesize = sysconf(_SC_PAGE_SIZE); + nidxdb->pagesize = rpmxdbPagesize(idxdb->xdb); /* calculate nslots the hard way, don't trust usedslots */ nslots = 0; @@ -906,9 +906,8 @@ int rpmidxOpenXdb(rpmidxdb *idxdbp, rpmpkgdb pkgdb, rpmxdb xdb, unsigned int xdb idxdb->xdbtag = xdbtag; idxdb->xdbid = id; idxdb->pkgdb = pkgdb; - idxdb->pagesize = sysconf(_SC_PAGE_SIZE); - if (rpmxdbIsRdonly(xdb)) - idxdb->rdonly = 1; + idxdb->pagesize = rpmxdbPagesize(xdb); + idxdb->rdonly = rpmxdbIsRdonly(xdb) ? 1 : 0; if (!id) { if (rpmidxInit(idxdb)) { free(idxdb); diff --git a/lib/backend/ndb/rpmxdb.c b/lib/backend/ndb/rpmxdb.c index f7c6ebd91b..c67c80cfde 100644 --- a/lib/backend/ndb/rpmxdb.c +++ b/lib/backend/ndb/rpmxdb.c @@ -1151,6 +1151,11 @@ int rpmxdbIsRdonly(rpmxdb xdb) return xdb->rdonly; } +unsigned int rpmxdbPagesize(rpmxdb xdb) +{ + return xdb->pagesize; +} + static int rpmxdbFsync(rpmxdb xdb) { #ifdef HAVE_FDATASYNC diff --git a/lib/backend/ndb/rpmxdb.h b/lib/backend/ndb/rpmxdb.h index 4358536ef1..88a3e617a6 100644 --- a/lib/backend/ndb/rpmxdb.h +++ b/lib/backend/ndb/rpmxdb.h @@ -7,6 +7,7 @@ int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pkgdb, const char *filename, int flags, in void rpmxdbClose(rpmxdb xdb); void rpmxdbSetFsync(rpmxdb xdb, int dofsync); int rpmxdbIsRdonly(rpmxdb xdb); +unsigned int rpmxdbPagesize(rpmxdb xdb); int rpmxdbLock(rpmxdb xdb, int excl); int rpmxdbUnlock(rpmxdb xdb, int excl); From 920bba45f99d59206041433b4d252f1318ca4c35 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Mon, 21 Oct 2019 11:52:52 +0200 Subject: [PATCH 09/54] Multiple fixes in rpmxdb.c for the ndb database backend Found by torture-testing the ndb database. * Usedslots was allocated with the wrong size This did not matter for rpm because rpm uses only a small number of index databases * The addslotpage function did not enqueue the new free slots correctly It never gets called in rpm * The protection bits were not set if moveblobto needed to map a blob This can happen if it is called from the moveblobstofront() function, it will just not shrink the database in the current call. (cherry picked from commit 62e6a750710293dde19d6de5e804f22601238b01) --- lib/backend/ndb/rpmxdb.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/backend/ndb/rpmxdb.c b/lib/backend/ndb/rpmxdb.c index c67c80cfde..2f94491e97 100644 --- a/lib/backend/ndb/rpmxdb.c +++ b/lib/backend/ndb/rpmxdb.c @@ -274,7 +274,7 @@ static int rpmxdbReadHeader(rpmxdb xdb) xdb->firstfree = 0; nslots = slotnpages * (pagesize / SLOT_SIZE) - SLOT_START + 1; slots = xcalloc(nslots + 1, sizeof(struct xdb_slot)); - usedslots = xcalloc(nslots + 1, sizeof(int)); + usedslots = xcalloc(nslots + 1, sizeof(struct xdb_slot *)); nused = 0; slotno = 1; slot = slots + 1; @@ -615,6 +615,7 @@ static int moveblobto(rpmxdb xdb, struct xdb_slot *oldslot, struct xdb_slot *aft didmap = 0; oldpagecnt = oldslot->pagecnt; if (!oldslot->mapped && oldpagecnt) { + oldslot->mapflags = PROT_READ; if (mapslot(xdb, oldslot)) return RPMRC_FAIL; didmap = 1; @@ -786,13 +787,16 @@ static int addslotpage(rpmxdb xdb) *slot = xdb->slots[nslots]; slot->slotno = nslots + spp; xdb->slots[slot->prev].next = slot->slotno; + + /* we have a new slotpage */ xdb->nslots += spp; + xdb->slots[0].pagecnt++; /* add new free slots to the firstfree chain */ memset(xdb->slots + nslots, 0, sizeof(*slot) * spp); for (i = 0; i < spp - 1; i++) { xdb->slots[nslots + i].slotno = nslots + i; - xdb->slots[nslots + i].next = i + 1; + xdb->slots[nslots + i].next = nslots + i + 1; } xdb->slots[nslots + i].slotno = nslots + i; xdb->firstfree = nslots; From 1440ece16701958de6573b7115b1d0f8fdfc1f4c Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Thu, 12 Sep 2019 13:55:41 +0300 Subject: [PATCH 10/54] Log debug messages upon entering and exiting chroot (cherry picked from commit 858d6babd6d9af3c91a152c3abc67122f80990e6) --- lib/rpmchroot.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/rpmchroot.c b/lib/rpmchroot.c index 88be3c2124..efaa74fd02 100644 --- a/lib/rpmchroot.c +++ b/lib/rpmchroot.c @@ -124,6 +124,7 @@ int rpmChrootIn(void) if (!_rpm_nouserns && getuid()) try_become_root(); + rpmlog(RPMLOG_DEBUG, "entering chroot %s\n", rootState.rootDir); if (chdir("/") == 0 && chroot(rootState.rootDir) == 0) { rootState.chrootDone = 1; } else { @@ -149,6 +150,7 @@ int rpmChrootOut(void) if (rootState.chrootDone > 1) { rootState.chrootDone--; } else if (rootState.chrootDone == 1) { + rpmlog(RPMLOG_DEBUG, "exiting chroot %s\n", rootState.rootDir); if (chroot(".") == 0 && fchdir(rootState.cwd) == 0) { rootState.chrootDone = 0; } else { From cc03a0c66fcfe8097ad82f7e949d20e9e4184770 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Tue, 17 Sep 2019 15:18:19 +0200 Subject: [PATCH 11/54] Print an error for expressions with missing operands Expressions like '5 +' did not print an error message before. (cherry picked from commit 1d055ae1df90fe54cf5c056de085cd987f0f6bbf) --- rpmio/expression.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rpmio/expression.c b/rpmio/expression.c index 193ae303ca..0f10a01a21 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -401,6 +401,11 @@ static Value doPrimary(ParseState state) v = valueMakeInteger(! v->data.i); break; + + case TOK_EOF: + exprErr(&state, _("unexpected end of expression"), NULL); + goto err; + default: goto err; break; From 54aa3efcfe3c5f88090c6c89e75bf9a711b3b6fd Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Wed, 18 Sep 2019 10:56:11 +0300 Subject: [PATCH 12/54] Fix type error introduced in commit 1d055ae1df90fe54cf5c056de085cd987f0f6bbf "state" is a pointer in this function, we don't want a pointer to it. Fixes garbage getting printed in the error message. Add a testcase too. (cherry picked from commit e79d1feb6d8111c462eb13403cf0b7ca36b09ff1) --- rpmio/expression.c | 2 +- tests/rpmmacro.at | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rpmio/expression.c b/rpmio/expression.c index 0f10a01a21..7c30fe711f 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -403,7 +403,7 @@ static Value doPrimary(ParseState state) break; case TOK_EOF: - exprErr(&state, _("unexpected end of expression"), NULL); + exprErr(state, _("unexpected end of expression"), NULL); goto err; default: diff --git a/tests/rpmmacro.at b/tests/rpmmacro.at index b425a6980a..9af506e62b 100644 --- a/tests/rpmmacro.at +++ b/tests/rpmmacro.at @@ -273,6 +273,7 @@ AT_CHECK([ runroot rpm --eval '%{expr:a*1}' runroot rpm --eval '%{expr:(5+1)*4)}' runroot rpm --eval '%{expr:a=!b}' +runroot rpm --eval '%{expr:4+}' ], [1], [], @@ -281,6 +282,7 @@ error: syntax error in expression: (5+1)*4) error: ^ error: syntax error while parsing ==: a=!b error: ^ +error: unexpected end of expression: 4+ ]) AT_CLEANUP From 46c4c4cdc8aa8574b0a323ebebddfb5e871a3066 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Wed, 18 Sep 2019 13:23:45 +0300 Subject: [PATCH 13/54] Trap division by zero in expression parser (cherry picked from commit 715ce7ce2eb6028553f8ce268a2f6cc76b9d5251) (cherry picked from commit 7c2900e682c95e89d0fc7daf5f4040fd3deb0241) --- rpmio/expression.c | 4 ++++ tests/rpmmacro.at | 2 ++ 2 files changed, 6 insertions(+) diff --git a/rpmio/expression.c b/rpmio/expression.c index 7c30fe711f..4221bb9e33 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -453,6 +453,10 @@ static Value doMultiplyDivide(ParseState state) if (valueIsInteger(v1)) { int i1 = v1->data.i, i2 = v2->data.i; + if ((i2 == 0) && (op == TOK_DIVIDE)) { + exprErr(state, _("division by zero"), NULL); + goto err; + } valueFree(v1); if (op == TOK_MULTIPLY) v1 = valueMakeInteger(i1 * i2); diff --git a/tests/rpmmacro.at b/tests/rpmmacro.at index 9af506e62b..930672203e 100644 --- a/tests/rpmmacro.at +++ b/tests/rpmmacro.at @@ -274,6 +274,7 @@ runroot rpm --eval '%{expr:a*1}' runroot rpm --eval '%{expr:(5+1)*4)}' runroot rpm --eval '%{expr:a=!b}' runroot rpm --eval '%{expr:4+}' +runroot rpm --eval '%{expr:1/0}' ], [1], [], @@ -283,6 +284,7 @@ error: ^ error: syntax error while parsing ==: a=!b error: ^ error: unexpected end of expression: 4+ +error: division by zero: 1/0 ]) AT_CLEANUP From 3d348d9b8db664bbef7fa7368e530a23add5955b Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 13 Sep 2019 16:13:57 +0200 Subject: [PATCH 14/54] Do not expand %{expr:} again after evaluating the expression (cherry picked from commit 7659f40cfe2f78d3a256e2952a0a4f91ff21eb88) --- rpmio/macro.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rpmio/macro.c b/rpmio/macro.c index 3e6f1d2abe..0b9f813bed 100644 --- a/rpmio/macro.c +++ b/rpmio/macro.c @@ -442,7 +442,6 @@ static void mbAppend(MacroBuf mb, char c) mb->nb--; } -#ifdef WITH_LUA static void mbAppendStr(MacroBuf mb, const char *str) { size_t len = strlen(str); @@ -454,7 +453,6 @@ static void mbAppendStr(MacroBuf mb, const char *str) mb->tpos += len; mb->nb -= len; } -#endif static const char * doDnl(MacroBuf mb, const char * se, size_t slen) { @@ -1050,6 +1048,7 @@ doFoo(MacroBuf mb, int chkexist, int negate, const char * f, size_t fn, int c; int verbose = (rpmIsVerbose() != 0); int expand = (g != NULL && gn > 0); + int expandagain = 1; /* Don't expand %{verbose:...} argument on false condition */ if (STREQ("verbose", f, fn) && (verbose == negate)) @@ -1107,6 +1106,7 @@ doFoo(MacroBuf mb, int chkexist, int negate, const char * f, size_t fn, if (expr) { free(buf); b = buf = expr; + expandagain = 0; } else { mb->error = 1; } @@ -1184,7 +1184,11 @@ doFoo(MacroBuf mb, int chkexist, int negate, const char * f, size_t fn, } if (b) { - (void) expandMacro(mb, b, 0); + if (expandagain) { + (void) expandMacro(mb, b, 0); + } else { + mbAppendStr(mb, b); + } } free(buf); } From f38b445340e7c19e427ae579af30ed1b535351a3 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 20 Sep 2019 10:32:08 +0300 Subject: [PATCH 15/54] Ensure expression syntax errors get at least a generic error message (cherry picked from commit f008f8c70a586e7687f4e71c2df557a0e5be56df) --- rpmio/expression.c | 1 + 1 file changed, 1 insertion(+) diff --git a/rpmio/expression.c b/rpmio/expression.c index 4221bb9e33..d6b0194503 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -407,6 +407,7 @@ static Value doPrimary(ParseState state) goto err; default: + exprErr(state, _("syntax error in expression"), state->p); goto err; break; } From 921faff6b02c93f387abc2f52634a0267b020eb2 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Mon, 23 Sep 2019 13:13:15 +0200 Subject: [PATCH 16/54] Free memory leak in unary op handling Also reduce the number of alloc/free calls by adding ValueSetXXX() variants that change a value. (cherry picked from commit 765fa386bd462432a3836f970b65366ed8b6ae00) --- rpmio/expression.c | 48 ++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/rpmio/expression.c b/rpmio/expression.c index d6b0194503..64715fa164 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -66,6 +66,26 @@ static Value valueMakeString(char *s) return v; } +/** + */ +static void valueSetInteger(Value v, int i) +{ + if (v->type == VALUE_TYPE_STRING) + v->data.s = _free(v->data.s); + v->type = VALUE_TYPE_INTEGER; + v->data.i = i; +} + +/** + */ +static void valueSetString(Value v, char *s) +{ + if (v->type == VALUE_TYPE_STRING) + v->data.s = _free(v->data.s); + v->type = VALUE_TYPE_STRING; + v->data.s = s; +} + /** */ static void valueFree( Value v) @@ -383,7 +403,7 @@ static Value doPrimary(ParseState state) goto err; } - v = valueMakeInteger(- v->data.i); + valueSetInteger(v, - v->data.i); break; case TOK_NOT: @@ -399,7 +419,7 @@ static Value doPrimary(ParseState state) goto err; } - v = valueMakeInteger(! v->data.i); + valueSetInteger(v, ! v->data.i); break; case TOK_EOF: @@ -458,11 +478,10 @@ static Value doMultiplyDivide(ParseState state) exprErr(state, _("division by zero"), NULL); goto err; } - valueFree(v1); if (op == TOK_MULTIPLY) - v1 = valueMakeInteger(i1 * i2); + valueSetInteger(v1, i1 * i2); else - v1 = valueMakeInteger(i1 / i2); + valueSetInteger(v1, i1 / i2); } else { exprErr(state, _("* and / not supported for strings"), NULL); goto err; @@ -511,11 +530,10 @@ static Value doAddSubtract(ParseState state) if (valueIsInteger(v1)) { int i1 = v1->data.i, i2 = v2->data.i; - valueFree(v1); if (op == TOK_ADD) - v1 = valueMakeInteger(i1 + i2); + valueSetInteger(v1, i1 + i2); else - v1 = valueMakeInteger(i1 - i2); + valueSetInteger(v1, i1 - i2); } else { char *copy; @@ -527,8 +545,7 @@ static Value doAddSubtract(ParseState state) copy = xmalloc(strlen(v1->data.s) + strlen(v2->data.s) + 1); (void) stpcpy( stpcpy(copy, v1->data.s), v2->data.s); - valueFree(v1); - v1 = valueMakeString(copy); + valueSetString(v1, copy); } } @@ -595,8 +612,7 @@ static Value doRelational(ParseState state) default: break; } - valueFree(v1); - v1 = valueMakeInteger(r); + valueSetInteger(v1, r); } else { const char * s1 = v1->data.s; const char * s2 = v2->data.s; @@ -623,8 +639,7 @@ static Value doRelational(ParseState state) default: break; } - valueFree(v1); - v1 = valueMakeInteger(r); + valueSetInteger(v1, r); } } @@ -671,11 +686,10 @@ static Value doLogical(ParseState state) if (valueIsInteger(v1)) { int i1 = v1->data.i, i2 = v2->data.i; - valueFree(v1); if (op == TOK_LOGICAL_AND) - v1 = valueMakeInteger(i1 && i2); + valueSetInteger(v1, i1 && i2); else - v1 = valueMakeInteger(i1 || i2); + valueSetInteger(v1, i1 || i2); } else { exprErr(state, _("&& and || not supported for strings"), NULL); goto err; From 2c15985848449a82bfe329cee65f07438090cf7f Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Tue, 3 Sep 2019 10:19:02 +0200 Subject: [PATCH 17/54] Disable marker on multiline expression error messages If an expression multiline like: %{expr: 0 || 0 || 0 || 0 |o| 0 || 0 || 0 || 0 } then it is better not to support marker pointing to the exact place of the error. (cherry picked from commit 076e3ea2a3ec121c62298a2b6ec9d4750b68bf05) --- rpmio/expression.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rpmio/expression.c b/rpmio/expression.c index 64715fa164..ad0199df51 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -130,6 +130,11 @@ typedef struct _parseState { static void exprErr(const struct _parseState *state, const char *msg, const char *p) { + const char *newLine = strchr(state->str,'\n'); + + if (newLine && (*(newLine+1) != '\0')) + p = NULL; + rpmlog(RPMLOG_ERR, "%s: %s\n", msg, state->str); if (p) { int l = p - state->str + strlen(msg) + 2; From 3fe2e1cec986ba7bdbf99beded9e4d9fa97b4b3d Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Fri, 30 Aug 2019 09:15:51 +0200 Subject: [PATCH 18/54] Correct and update Query formats documentation - in a majority of examples in the manual the text after the --queryformat argument is in double quotes - thus it does not look good to wrote that "A query format is passed to RPM after the --queryformat argument, and normally should be enclosed in single quotes". - in the case of: rpm -qa -i --queryformat "%{NAME} %{SIZE}\n" -i is not ignored, so this is omitted in the text. - language correction - added a link to a formatting tags documentation - added an explicit example of a query expression - removed example of "viewing the verify flags', that does not have any purpose here (cherry picked from commit 0cb0e69eafa069c2a6150ae2f573a4373ab058d1) --- doc/manual/queryformat | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/doc/manual/queryformat b/doc/manual/queryformat index cf7d9fc119..4bf35a21a7 100644 --- a/doc/manual/queryformat +++ b/doc/manual/queryformat @@ -38,9 +38,8 @@ be a string or number only. \section queryformat_format Query Formats A query format is passed to RPM after the --queryformat argument, and normally -should be enclosed in single quotes. This query format is then used to print -the information section of a query. This means that when both -i and ---queryformat are used in a command, the -i is essentially ignored. +should be enclosed in quotes. This query format is then used to print +the information section of a query. The query format is similar to a C style printf string, which the printf(2) man page provides a good introduction to. However, as RPM already knows the @@ -90,11 +89,11 @@ grep or awk). If you try the obvious, rpm -q --queryformat "[%{NAME} %{FILENAMES}\n]" cdp \endverbatim -If you try this, you'll see RPM complain about an "array iterator used -with different sized arrays". Internally, all items in RPM are actually -arrays, so the NAME is a string array containing one element. When you -tell RPM to iterate over the NAME and FILENAMES elements, RPM notices -the two tags have different numbers of elements and complains. +you'll see RPM complain about an "array iterator used with different +sized arrays". Internally, all items in RPM are actually arrays, so the +NAME is a string array containing one element. When you tell RPM to iterate +over the NAME and FILENAMES elements, RPM notices the two tags have +different numbers of elements and complains. To make this work properly, you need to tell RPM to always print the first item in the NAME element. You do this by placing a '=' before the tag @@ -136,6 +135,9 @@ readable format in SI resp IEC 80000 standard. humansi uses 1K = 1000, 1M = 1000000, ... humaniec uses 1K = 1024, 1M = 1048576, ... +The formatting tags are documented in rpm (8) manual in the QUERY OPTIONS +section. + \section queryformat_expressions Query Expressions Simple conditionals may be evaluated through query expressions. Expressions @@ -151,26 +153,8 @@ the SOMETAG tag is present, and "missing" otherwise: Notice that the subformats "present" and "missing" must be inside of curly braces. -\section queryformat_example Example: Viewing the Verify Flags - -The following example query is run against dev because I know %verify -is used there. -\verbatim - rpm -q --qf '[%{filenames} %{fileverifyflags}\n]' dev -\endverbatim - -The flags are defined in rpmfiles.h (check there for changes): \verbatim - #define RPMVERIFY_MD5 (1 << 0) - #define RPMVERIFY_FILESIZE (1 << 1) - #define RPMVERIFY_LINKTO (1 << 2) - #define RPMVERIFY_USER (1 << 3) - #define RPMVERIFY_GROUP (1 << 4) - #define RPMVERIFY_MTIME (1 << 5) - #define RPMVERIFY_MODE (1 << 6) - #define RPMVERIFY_RDEV (1 << 7) + rpm -qa --queryformat "%-30{NAME} %|PREINPROG?{%{PREINPROG}}:{no}|\n" \endverbatim -A 1 bit in the output of the query means the check is enabled. - */ From bb06a993d4bd2379e47da8a3832d029255b531b8 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 30 Aug 2019 11:47:33 +0200 Subject: [PATCH 19/54] Support libgrypt as crypto library (cherry picked from commit 037106ecc899bad6d6e6f9d95768699542b871ea) --- configure.ac | 21 +- rpmio/Makefile.am | 6 + rpmio/digest_libgcrypt.c | 404 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 rpmio/digest_libgcrypt.c diff --git a/configure.ac b/configure.ac index 62703ee135..cf6674c58b 100644 --- a/configure.ac +++ b/configure.ac @@ -270,7 +270,7 @@ AM_CONDITIONAL(LIBDWARF,[test "$WITH_LIBDWARF" = yes]) # Select crypto library AC_ARG_WITH(crypto, [AC_HELP_STRING([--with-crypto=CRYPTO_LIB], - [The cryptographic library to use (nss|beecrypt|openssl). The default is nss.]) + [The cryptographic library to use (nss|beecrypt|openssl|libgcrypt). The default is nss.]) ],[], [with_crypto=nss]) @@ -388,6 +388,25 @@ AM_CONDITIONAL([WITH_OPENSSL],[test "$with_crypto" = openssl]) AC_SUBST(WITH_OPENSSL_INCLUDE) AC_SUBST(WITH_OPENSSL_LIB) +#================= +# Check for libgcrypt library. +WITH_LIBGCRYPT_INCLUDE= +WITH_LIBGCRYPT_LIB= +if test "$with_crypto" = libgcrypt ; then +AC_PATH_TOOL(LIBGCRYPT_CONFIG, libgcrypt-config, notfound) +if test notfound != "$LIBGCRYPT_CONFIG" ; then +WITH_LIBGCRYPT_INCLUDE=`$LIBGCRYPT_CONFIG --cflags` +WITH_LIBGCRYPT_LIB=`$LIBGCRYPT_CONFIG --libs` +fi +if test -z "$WITH_LIBGCRYPT_LIB" ; then +AC_MSG_ERROR([libgcrypt not found]) +fi +fi + +AM_CONDITIONAL([WITH_LIBGCRYPT],[test "$with_crypto" = libgcrypt]) +AC_SUBST(WITH_LIBGCRYPT_INCLUDE) +AC_SUBST(WITH_LIBGCRYPT_LIB) + #================= # Check for NSS library. # We need nss.h from NSS which needs nspr.h. Unfortunately both glibc and NSS diff --git a/rpmio/Makefile.am b/rpmio/Makefile.am index a093e31ad5..63de85d89e 100644 --- a/rpmio/Makefile.am +++ b/rpmio/Makefile.am @@ -7,6 +7,7 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir) -I$(top_builddir)/include/ AM_CPPFLAGS += @WITH_NSS_INCLUDE@ AM_CPPFLAGS += @WITH_BEECRYPT_INCLUDE@ AM_CPPFLAGS += @WITH_OPENSSL_INCLUDE@ +AM_CPPFLAGS += @WITH_LIBGCRYPT_INCLUDE@ AM_CPPFLAGS += @WITH_POPT_INCLUDE@ AM_CPPFLAGS += $(ZSTD_CFLAGS) AM_CPPFLAGS += -I$(top_srcdir)/misc @@ -29,9 +30,13 @@ else if WITH_OPENSSL librpmio_la_SOURCES += digest_openssl.c else +if WITH_LIBGCRYPT +librpmio_la_SOURCES += digest_libgcrypt.c +else librpmio_la_SOURCES += digest_nss.c endif endif +endif librpmio_la_LDFLAGS = -version-info $(rpm_version_info) @@ -40,6 +45,7 @@ librpmio_la_LIBADD = \ @WITH_NSS_LIB@ \ @WITH_BEECRYPT_LIB@ \ @WITH_OPENSSL_LIB@ \ + @WITH_LIBGCRYPT_LIB@ \ @WITH_BZ2_LIB@ \ @WITH_ZLIB_LIB@ \ @WITH_POPT_LIB@ \ diff --git a/rpmio/digest_libgcrypt.c b/rpmio/digest_libgcrypt.c new file mode 100644 index 0000000000..b31fda5690 --- /dev/null +++ b/rpmio/digest_libgcrypt.c @@ -0,0 +1,404 @@ +#include "system.h" + +#include + +#include +#include "rpmio/digest.h" +#include "rpmio/rpmio_internal.h" +#include "debug.h" + +/** + * MD5/SHA1 digest private data. + */ +struct DIGEST_CTX_s { + rpmDigestFlags flags; /*!< Bit(s) to control digest operation. */ + int algo; /*!< Used hash algorithm */ + gcry_md_hd_t h; +}; + + +/**************************** init ************************************/ + +int rpmInitCrypto(void) { + return 0; +} + +int rpmFreeCrypto(void) { + return 0; +} + +/**************************** digest ************************************/ + +size_t rpmDigestLength(int hashalgo) +{ + switch (hashalgo) { + case PGPHASHALGO_MD5: + return 16; + case PGPHASHALGO_SHA1: + return 20; + case PGPHASHALGO_SHA224: + return 28; + case PGPHASHALGO_SHA256: + return 32; + case PGPHASHALGO_SHA384: + return 48; + case PGPHASHALGO_SHA512: + return 64; + default: + return 0; + } +} + +static int hashalgo2gcryalgo(int hashalgo) +{ + switch (hashalgo) { + case PGPHASHALGO_MD5: + return GCRY_MD_MD5; + case PGPHASHALGO_SHA1: + return GCRY_MD_SHA1; + case PGPHASHALGO_SHA224: + return GCRY_MD_SHA224; + case PGPHASHALGO_SHA256: + return GCRY_MD_SHA256; + case PGPHASHALGO_SHA384: + return GCRY_MD_SHA384; + case PGPHASHALGO_SHA512: + return GCRY_MD_SHA512; + default: + return 0; + } +} + +DIGEST_CTX rpmDigestInit(int hashalgo, rpmDigestFlags flags) +{ + gcry_md_hd_t h; + DIGEST_CTX ctx; + int gcryalgo = hashalgo2gcryalgo(hashalgo); + + if (!gcryalgo || gcry_md_open(&h, gcryalgo, 0) != 0) + return NULL; + + ctx = xcalloc(1, sizeof(*ctx)); + ctx->flags = flags; + ctx->algo = hashalgo; + ctx->h = h; + return ctx; +} + +int rpmDigestUpdate(DIGEST_CTX ctx, const void * data, size_t len) +{ + if (ctx == NULL) + return -1; + gcry_md_write(ctx->h, data, len); + return 0; +} + +int rpmDigestFinal(DIGEST_CTX ctx, void ** datap, size_t *lenp, int asAscii) +{ + unsigned char *digest; + int digestlen; + if (ctx == NULL) + return -1; + digest = gcry_md_read(ctx->h, 0); + digestlen = rpmDigestLength(ctx->algo); + if (!asAscii) { + if (lenp) + *lenp = digestlen; + if (datap) { + *datap = xmalloc(digestlen); + memcpy(*datap, digest, digestlen); + } + } else { + if (lenp) + *lenp = 2 * digestlen + 1; + if (datap) { + *datap = pgpHexStr((const uint8_t *)digest, digestlen); + } + } + gcry_md_close(ctx->h); + free(ctx); + return 0; +} + +DIGEST_CTX rpmDigestDup(DIGEST_CTX octx) +{ + DIGEST_CTX nctx = NULL; + if (octx) { + gcry_md_hd_t h; + if (gcry_md_copy(&h, octx->h)) + return NULL; + nctx = memcpy(xcalloc(1, sizeof(*nctx)), octx, sizeof(*nctx)); + nctx->h = h; + } + return nctx; +} + + +/****************************** RSA **************************************/ + +struct pgpDigSigRSA_s { + gcry_mpi_t s; +}; + +struct pgpDigKeyRSA_s { + gcry_mpi_t n; + gcry_mpi_t e; +}; + +static int pgpSetSigMpiRSA(pgpDigAlg pgpsig, int num, const uint8_t *p) +{ + struct pgpDigSigRSA_s *sig = pgpsig->data; + int mlen = pgpMpiLen(p); + int rc = 1; + + if (!sig) + sig = pgpsig->data = xcalloc(1, sizeof(*sig)); + + switch (num) { + case 0: + if (!gcry_mpi_scan(&sig->s, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + } + return rc; +} + +static int pgpSetKeyMpiRSA(pgpDigAlg pgpkey, int num, const uint8_t *p) +{ + struct pgpDigKeyRSA_s *key = pgpkey->data; + int mlen = pgpMpiLen(p); + int rc = 1; + + if (!key) + key = pgpkey->data = xcalloc(1, sizeof(*key)); + + switch (num) { + case 0: + if (!gcry_mpi_scan(&key->n, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + case 1: + if (!gcry_mpi_scan(&key->e, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + } + return rc; +} + +static int pgpVerifySigRSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig, uint8_t *hash, size_t hashlen, int hash_algo) +{ + struct pgpDigKeyRSA_s *key = pgpkey->data; + struct pgpDigSigRSA_s *sig = pgpsig->data; + gcry_sexp_t sexp_sig = NULL, sexp_data = NULL, sexp_pkey = NULL; + const char *hash_algo_name; + int rc = 1; + + if (!sig || !key) + return rc; + + hash_algo_name = gcry_md_algo_name(hashalgo2gcryalgo(hash_algo)); + gcry_sexp_build(&sexp_sig, NULL, "(sig-val (rsa (s %M)))", sig->s); + gcry_sexp_build(&sexp_data, NULL, "(data (flags pkcs1) (hash %s %b))", hash_algo_name, (int)hashlen, (const char *)hash); + gcry_sexp_build(&sexp_pkey, NULL, "(public-key (rsa (n %M) (e %M)))", key->n, key->e); + if (sexp_sig && sexp_data && sexp_pkey) + rc = gcry_pk_verify(sexp_sig, sexp_data, sexp_pkey) == 0 ? 0 : 1; + gcry_sexp_release(sexp_sig); + gcry_sexp_release(sexp_data); + gcry_sexp_release(sexp_pkey); + return rc; +} + +static void pgpFreeSigRSA(pgpDigAlg pgpsig) +{ + struct pgpDigSigRSA_s *sig = pgpsig->data; + if (sig) { + gcry_mpi_release(sig->s); + pgpsig->data = _free(sig); + } +} + +static void pgpFreeKeyRSA(pgpDigAlg pgpkey) +{ + struct pgpDigKeyRSA_s *key = pgpkey->data; + if (key) { + gcry_mpi_release(key->n); + gcry_mpi_release(key->e); + pgpkey->data = _free(key); + } +} + + +/****************************** DSA **************************************/ + +struct pgpDigSigDSA_s { + gcry_mpi_t r; + gcry_mpi_t s; +}; + +struct pgpDigKeyDSA_s { + gcry_mpi_t p; + gcry_mpi_t q; + gcry_mpi_t g; + gcry_mpi_t y; +}; + +static int pgpSetSigMpiDSA(pgpDigAlg pgpsig, int num, const uint8_t *p) +{ + struct pgpDigSigDSA_s *sig = pgpsig->data; + int mlen = pgpMpiLen(p); + int rc = 1; + + if (!sig) + sig = pgpsig->data = xcalloc(1, sizeof(*sig)); + + switch (num) { + case 0: + if (!gcry_mpi_scan(&sig->r, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + case 1: + if (!gcry_mpi_scan(&sig->s, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + } + return rc; +} + +static int pgpSetKeyMpiDSA(pgpDigAlg pgpkey, int num, const uint8_t *p) +{ + struct pgpDigKeyDSA_s *key = pgpkey->data; + int mlen = pgpMpiLen(p); + int rc = 1; + + if (!key) + key = pgpkey->data = xcalloc(1, sizeof(*key)); + + switch (num) { + case 0: + if (!gcry_mpi_scan(&key->p, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + case 1: + if (!gcry_mpi_scan(&key->q, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + case 2: + if (!gcry_mpi_scan(&key->g, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + case 3: + if (!gcry_mpi_scan(&key->y, GCRYMPI_FMT_PGP, p, mlen, NULL)) + rc = 0; + break; + } + return rc; +} + +static int pgpVerifySigDSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig, uint8_t *hash, size_t hashlen, int hash_algo) +{ + struct pgpDigKeyDSA_s *key = pgpkey->data; + struct pgpDigSigDSA_s *sig = pgpsig->data; + gcry_sexp_t sexp_sig = NULL, sexp_data = NULL, sexp_pkey = NULL; + int rc = 1; + + if (!sig || !key) + return rc; + + gcry_sexp_build(&sexp_sig, NULL, "(sig-val (dsa (r %M) (s %M)))", sig->r, sig->s); + gcry_sexp_build(&sexp_data, NULL, "(data (flags raw) (value %b))", (int)hashlen, (const char *)hash); + gcry_sexp_build(&sexp_pkey, NULL, "(public-key (dsa (p %M) (q %M) (g %M) (y %M)))", key->p, key->q, key->g, key->y); + if (sexp_sig && sexp_data && sexp_pkey) + rc = gcry_pk_verify(sexp_sig, sexp_data, sexp_pkey) == 0 ? 0 : 1; + gcry_sexp_release(sexp_sig); + gcry_sexp_release(sexp_data); + gcry_sexp_release(sexp_pkey); + return rc; +} + +static void pgpFreeSigDSA(pgpDigAlg pgpsig) +{ + struct pgpDigSigDSA_s *sig = pgpsig->data; + if (sig) { + gcry_mpi_release(sig->r); + gcry_mpi_release(sig->s); + pgpsig->data = _free(sig); + } +} + +static void pgpFreeKeyDSA(pgpDigAlg pgpkey) +{ + struct pgpDigKeyDSA_s *key = pgpkey->data; + if (key) { + gcry_mpi_release(key->p); + gcry_mpi_release(key->q); + gcry_mpi_release(key->g); + gcry_mpi_release(key->y); + pgpkey->data = _free(key); + } +} + + +/****************************** NULL **************************************/ + +static int pgpSetMpiNULL(pgpDigAlg pgpkey, int num, const uint8_t *p) +{ + return 1; +} + +static int pgpVerifyNULL(pgpDigAlg pgpkey, pgpDigAlg pgpsig, + uint8_t *hash, size_t hashlen, int hash_algo) +{ + return 1; +} + +pgpDigAlg pgpPubkeyNew(int algo) +{ + pgpDigAlg ka = xcalloc(1, sizeof(*ka));; + + switch (algo) { + case PGPPUBKEYALGO_RSA: + ka->setmpi = pgpSetKeyMpiRSA; + ka->free = pgpFreeKeyRSA; + ka->mpis = 2; + break; + case PGPPUBKEYALGO_DSA: + ka->setmpi = pgpSetKeyMpiDSA; + ka->free = pgpFreeKeyDSA; + ka->mpis = 4; + break; + default: + ka->setmpi = pgpSetMpiNULL; + ka->mpis = -1; + break; + } + + ka->verify = pgpVerifyNULL; /* keys can't be verified */ + + return ka; +} + +pgpDigAlg pgpSignatureNew(int algo) +{ + pgpDigAlg sa = xcalloc(1, sizeof(*sa)); + + switch (algo) { + case PGPPUBKEYALGO_RSA: + sa->setmpi = pgpSetSigMpiRSA; + sa->free = pgpFreeSigRSA; + sa->verify = pgpVerifySigRSA; + sa->mpis = 1; + break; + case PGPPUBKEYALGO_DSA: + sa->setmpi = pgpSetSigMpiDSA; + sa->free = pgpFreeSigDSA; + sa->verify = pgpVerifySigDSA; + sa->mpis = 2; + break; + default: + sa->setmpi = pgpSetMpiNULL; + sa->verify = pgpVerifyNULL; + sa->mpis = -1; + break; + } + return sa; +} From ed275056afc6f79072505a9f8594b4b1b814dde8 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Wed, 4 Sep 2019 10:56:49 +0200 Subject: [PATCH 20/54] Improve description of conditionals in spec documentation (cherry picked from commit 07be45f2023b7b59d6880a9b577b339c439b1018) --- doc/manual/spec | 50 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/doc/manual/spec b/doc/manual/spec index bfb76a049a..4c3c7a9d52 100644 --- a/doc/manual/spec +++ b/doc/manual/spec @@ -216,22 +216,44 @@ can express this as \section conditionals Conditionals -RPM's spec file format allows conditional blocks of code to be used dependent -on various properties such as architecture (%ifarch / %ifnarch), operating system -(%ifos / %ifnos), or any other conditional expression (%if). Conditionals may be -nested within other conditionals: - -\verbatim - %ifarch ppc64 ppc x86_64 - ... - %if 0%{?ver} && 0%{?ver} >= 100 - ... - %else - ... - %endif +RPM's spec file format allows conditional blocks of code to be used +depending on various properties such as architecture (%ifarch /%ifnarch), +operating system (%ifos / %ifnos), or a conditional expression (%if). + +%ifarch is generally used for building RPM packages for multiple +platforms like: +\verbatim + %ifarch s390 s390x + BuildRequires: s390utils-devel %endif \endverbatim -Conditionals are not macros, thus they can be used in spec files only. +%ifos is used to control RPM's spec file processing according to the +build target operating system. + +%if can be used for various purposes. The test can be evaluated based on +the existence of a macro, like: +\verbatim + %if %{defined with_foo} && %{undefined with_bar} +\endverbatim +string comparison: +\verbatim + %if "%{optimize_flags}" != "none" +\endverbatim +or a mathematical statement: +\verbatim + %if 0%{?fedora} > 10 || 0%{?rhel} > 7 +\endverbatim +Generally, a mathematical statement allows to use logical operators +&&, ||, !, relational operators !=, ==, <, > , <=, >=, arithmetic operators ++, -, /, * and parentheses. + +The conditional blocks end by %endif. Inside the conditional block %elif, +%elifarch, %elifos or %else can be optionally used. Conditionals %endif and +%else should not be followed by any text. Conditionals may be nested within +other conditionals. + +%if-conditionals are not macros, and are unlikely to yield expected results +if used in them. */ From 5d002a1ae67df9ec5a2f37f40d5e01ce1a4d3db9 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Tue, 10 Sep 2019 12:11:39 +0200 Subject: [PATCH 21/54] Add description of comments in spec documentation (cherry picked from commit 22c26c7444c32dfe6b75b250b247b5d57f989ba6) --- doc/manual/spec | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/manual/spec b/doc/manual/spec index 4c3c7a9d52..a84047e97f 100644 --- a/doc/manual/spec +++ b/doc/manual/spec @@ -214,6 +214,28 @@ can express this as BuildConflicts: gcc <= 2.7.2.1 \endverbatim +\section comments Comments + +Comments in spec file have # at the start of the line. + +\verbatim + # this is a comment +\endverbatim + +Macros are expanded even in comment lines. If this is undesireable, escape +the macro with an extra percent sign (%): + +\verbatim + # make unversioned %%__python an error unless explicitly overridden +\endverbatim + +Another option is to use built-in macro %dnl that discards text to next +line without expanding it. + +\verbatim + %dnl make unversioned %__python an error unless explicitly overridden +\endverbatim + \section conditionals Conditionals RPM's spec file format allows conditional blocks of code to be used From 90228518edffb73fa653c95668ec9067bb01243b Mon Sep 17 00:00:00 2001 From: Shogo Matsumoto Date: Sun, 8 Sep 2019 22:44:51 -0400 Subject: [PATCH 22/54] Suppress inhibition lock warning message when DBus service is not available The message may just confuse users if DBus is not running as default, e.g. single-user mode. We suppress it when DBus is not available, which is done by checking two cases: socket does not exist (DBUS_ERROR_FILE_NOT_FOUND), or unable to connect to server (DBUS_ERROR_NO_SERVER). Note that this is an approximate but not an exact way to detect whether DBus service should be running in the environment or not. (cherry picked from commit 708e61307bc3fd027b016fdf5a1d1a5274c1843c) --- plugins/systemd_inhibit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/systemd_inhibit.c b/plugins/systemd_inhibit.c index 0628188baf..9cb7227614 100644 --- a/plugins/systemd_inhibit.c +++ b/plugins/systemd_inhibit.c @@ -52,7 +52,9 @@ static int inhibit(void) dbus_message_unref(reply); } - if (dbus_error_is_set(&err)) { + if (dbus_error_is_set(&err) + && !dbus_error_has_name(&err, DBUS_ERROR_NO_SERVER) + && !dbus_error_has_name(&err, DBUS_ERROR_FILE_NOT_FOUND)) { rpmlog(RPMLOG_WARNING, "Unable to get systemd shutdown inhibition lock: %s\n", err.message); From 467e0c7544b1ac4da4281cd9077a8d377ab9a6e7 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Wed, 11 Sep 2019 21:06:24 +0200 Subject: [PATCH 23/54] Add 'string' into query format extensions in man-pages (cherry picked from commit cead442ecf6f85521c9c64ba93ea44e0f084131d) --- doc/rpm.8 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/rpm.8 b/doc/rpm.8 index 27d5f3c02d..7004a50805 100644 --- a/doc/rpm.8 +++ b/doc/rpm.8 @@ -525,6 +525,9 @@ Display signature fingerprint and time. \fB:shescape\fR Escape single quotes for use in a script. .TP +\fB:string\fR +Display string format. (default) +.TP \fB:triggertype\fR Display trigger suffix. .TP From a753b37b596c13361e34c6c730cd68b0d35a5b92 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 20 Sep 2019 10:29:38 +0300 Subject: [PATCH 24/54] Always check for rdToken() return codes in expression parsing (cherry picked from commit 968c53cd3de01e16e8be3da800c8cd0d8e25fcbe) --- rpmio/expression.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rpmio/expression.c b/rpmio/expression.c index ad0199df51..e70ebb6bd6 100644 --- a/rpmio/expression.c +++ b/rpmio/expression.c @@ -714,7 +714,7 @@ int rpmExprBool(const char *expr) { struct _parseState state; int result = -1; - Value v; + Value v = NULL; DEBUG(printf("parseExprBoolean(?, '%s')\n", expr)); @@ -722,7 +722,8 @@ int rpmExprBool(const char *expr) state.p = state.str = xstrdup(expr); state.nextToken = 0; state.tokenValue = NULL; - (void) rdToken(&state); + if (rdToken(&state)) + goto exit; /* Parse the expression. */ v = doLogical(&state); @@ -758,7 +759,7 @@ char *rpmExprStr(const char *expr) { struct _parseState state; char *result = NULL; - Value v; + Value v = NULL; DEBUG(printf("parseExprString(?, '%s')\n", expr)); @@ -766,7 +767,8 @@ char *rpmExprStr(const char *expr) state.p = state.str = xstrdup(expr); state.nextToken = 0; state.tokenValue = NULL; - (void) rdToken(&state); + if (rdToken(&state)) + goto exit; /* Parse the expression. */ v = doLogical(&state); From e421ebce4cfed379d9dd94d0fbf3f051e69a08c8 Mon Sep 17 00:00:00 2001 From: Panu Matilainen Date: Fri, 20 Sep 2019 15:15:08 +0300 Subject: [PATCH 25/54] Resurrect %_missing_doc_files_terminate_build functionality Fixes regression from commit 1ba05a7456aafb52e89df5dd42d494d09f9ea6a4 where doc files always terminate build regardless of the macro value. Add a testcase to go. Fixes #807 (cherry picked from commit 9dff0b37bb3a6c6acbba6d5579d7a6fe03424683) --- build/files.c | 19 ++++++++++++++----- tests/Makefile.am | 1 + tests/data/SPECS/docmiss.spec | 24 ++++++++++++++++++++++++ tests/rpmbuild.at | 20 ++++++++++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 tests/data/SPECS/docmiss.spec diff --git a/build/files.c b/build/files.c index ad4f462f1b..6dfd801c89 100644 --- a/build/files.c +++ b/build/files.c @@ -1408,9 +1408,19 @@ static rpmRC addFile(FileList fl, const char * diskPath, statp = fakeStat(&(fl->cur), &statbuf); } else { int lvl = RPMLOG_ERR; + int ignore = 0; const char *msg = fl->cur.isDir ? _("Directory not found: %s\n") : _("File not found: %s\n"); - if (fl->cur.attrFlags & RPMFILE_EXCLUDE) { + if (fl->cur.attrFlags & RPMFILE_EXCLUDE) + ignore = 1; + if (fl->cur.attrFlags & RPMFILE_DOC) { + int strict_doc = + rpmExpandNumeric("%{?_missing_doc_files_terminate_build}"); + if (!strict_doc) + ignore = 1; + } + + if (ignore) { lvl = RPMLOG_WARNING; rc = RPMRC_OK; } @@ -2367,11 +2377,10 @@ static void processSpecialDir(rpmSpec spec, Package pkg, FileList fl, } if (install) { - rpmRC rc = doScript(spec, RPMBUILD_STRINGBUF, sdname, - getStringBuf(docScript), test, NULL); - - if (rc && rpmExpandNumeric("%{?_missing_doc_files_terminate_build}")) + if (doScript(spec, RPMBUILD_STRINGBUF, sdname, + getStringBuf(docScript), test, NULL)) { fl->processingFailed = 1; + } } basepath = rpmGenPath(spec->rootDir, "%{_builddir}", spec->buildSubdir); diff --git a/tests/Makefile.am b/tests/Makefile.am index 94ffd8da3c..38121eda1e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -41,6 +41,7 @@ EXTRA_DIST += $(TESTSUITE_AT) ## testsuite data EXTRA_DIST += data/SPECS/attrtest.spec EXTRA_DIST += data/SPECS/buildrequires.spec +EXTRA_DIST += data/SPECS/docmiss.spec EXTRA_DIST += data/SPECS/hello.spec EXTRA_DIST += data/SPECS/hello-auto.spec EXTRA_DIST += data/SPECS/hello-r2.spec diff --git a/tests/data/SPECS/docmiss.spec b/tests/data/SPECS/docmiss.spec new file mode 100644 index 0000000000..2c49370241 --- /dev/null +++ b/tests/data/SPECS/docmiss.spec @@ -0,0 +1,24 @@ +Name: docmiss +Version: 1.0 +Release: 1 +Summary: Testing missing doc behavior +Group: Testing +License: GPL + +%description +%{summary} + + +%prep +%setup -c -n %{name}-%{version} -T + +cat << EOF > COPYING +This is not a license +EOF + +cat << EOF > README +This is a dog project +EOF + +%files +%doc CREDITS COPYING README diff --git a/tests/rpmbuild.at b/tests/rpmbuild.at index 7d4e590b8e..eefa264286 100644 --- a/tests/rpmbuild.at +++ b/tests/rpmbuild.at @@ -1613,6 +1613,26 @@ run rpmbuild \ []) AT_CLEANUP +AT_SETUP([rpmbuild missing doc]) +AT_KEYWORDS([build]) +AT_CHECK_UNQUOTED([ +rm -rf ${TOPDIR} + +for val in 1 0; do + run rpmbuild \ + -bb --quiet \ + --define "_missing_doc_files_terminate_build ${val}" \ + "${abs_srcdir}"/data/SPECS/docmiss.spec + echo $? +done +], +[], +[1 +0 +], +[ignore]) +AT_CLEANUP + AT_SETUP([%if, %else, %elif test basic]) AT_KEYWORDS([if else elif build]) AT_CHECK([ From 2e6c4ecb93a9b3246467d070217f0c5aeac21781 Mon Sep 17 00:00:00 2001 From: Peter Jones Date: Thu, 19 Sep 2019 13:09:56 -0400 Subject: [PATCH 26/54] Add all of the rpmbuild macro aliases to rpmspec as well This adds all of the rpmbuild popt aliases that expand to defines to rpmspec as well. It also changes --trace to include --POPTdesc argument help. [v2: fix an error that broke rpmbuild --trace] Signed-off-by: Peter Jones (cherry picked from commit 1896e58ffdf2278c47fea5f6e7d29bbf81eac1ad) --- rpmpopt.in | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/rpmpopt.in b/rpmpopt.in index 7021898a42..8e4ef02751 100644 --- a/rpmpopt.in +++ b/rpmpopt.in @@ -221,8 +221,8 @@ rpmbuild alias --buildpolicy --define '__os_install_post %{_rpmconfigdir}/brp-!# rpmbuild alias --sign \ --pipe 'rpm --addsign `grep ".*: .*\.rpm$"|cut -d: -f2` < "/dev/"`ps -p $$ -o tty | tail -n 1`' \ --POPTdesc=$"generate GPG signature (deprecated, use command rpmsign instead)" -# [--trace] "trace macro expansion" -rpmbuild alias --trace --eval '%trace' +rpmbuild alias --trace --eval '%trace' \ + --POPTdesc=$"trace macro expansion" rpmbuild alias --nodebuginfo --define 'debug_package %{nil}' \ --POPTdesc=$"do not generate debuginfo for this package" @@ -249,7 +249,24 @@ rpmspec alias --buildconflicts --srpm --conflicts \ --POPTdesc=$"list capabilities conflicting with build of this package" rpmspec alias --buildrequires --srpm --requires \ --POPTdesc=$"list capabilities required to build this package" -# [--trace] "trace macro expansion" -rpmspec alias --trace --eval '%trace' +rpmspec alias --httpport --define '_httpport !#:+' +rpmspec alias --httpproxy --define '_httpproxy !#:+' +rpmspec alias --with --define "_with_!#:+ --with-!#:+" \ + --POPTdesc=$"enable configure