From 767015ae80e9c107ccf9f1b661af7c080d655426 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 10 Jan 2020 15:47:13 +0100 Subject: [PATCH 1/4] ndb: only clear the dbenv in the rpmdb if the last reference is gone Otherwise we will segfault if just one index is closed. --- lib/backend/ndb/glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backend/ndb/glue.c b/lib/backend/ndb/glue.c index 376e360e36..841c2fe421 100644 --- a/lib/backend/ndb/glue.c +++ b/lib/backend/ndb/glue.c @@ -52,8 +52,8 @@ static void closeEnv(rpmdb rdb) if (ndbenv->data) free(ndbenv->data); free(ndbenv); + rdb->db_dbenv = 0; } - rdb->db_dbenv = 0; } static struct ndbEnv_s *openEnv(rpmdb rdb) From 4bb77ea7e58b8b994cad6bf4f467977bc2c3b854 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 10 Jan 2020 15:49:16 +0100 Subject: [PATCH 2/4] ndb: add a rpmxdbDelAllBlobs method This will delete all blobs (i.e. index databases) from the ndb master index database. We do this kind of low-level on purpose, so that it even works if the index database is corrupt. This will be used in the next commit which implements automatic index regeneration if the index is out of sync. --- lib/backend/ndb/rpmxdb.c | 81 +++++++++++++++++++++++++++++++--------- lib/backend/ndb/rpmxdb.h | 1 + 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/lib/backend/ndb/rpmxdb.c b/lib/backend/ndb/rpmxdb.c index 2f94491e97..3baafa6166 100644 --- a/lib/backend/ndb/rpmxdb.c +++ b/lib/backend/ndb/rpmxdb.c @@ -224,11 +224,33 @@ static int usedslots_cmp(const void *a, const void *b) return sa->startpage > sb->startpage ? 1 : -1; } +static int rpmxdbReadHeaderRaw(rpmxdb xdb, unsigned int *generationp, unsigned int *slotnpagesp, unsigned int *pagesizep, unsigned int *usergenerationp) +{ + unsigned int header[XDB_HEADER_SIZE / sizeof(unsigned int)]; + unsigned int version; + if (pread(xdb->fd, header, sizeof(header), 0) != sizeof(header)) + return RPMRC_FAIL; + if (le2ha((unsigned char *)header + XDB_OFFSET_MAGIC) != XDB_MAGIC) + return RPMRC_FAIL; + version = le2ha((unsigned char *)header + XDB_OFFSET_VERSION); + if (version != XDB_VERSION) { + rpmlog(RPMLOG_ERR, _("rpmxdb: Version mismatch. Expected version: %u. " + "Found version: %u\n"), XDB_VERSION, version); + return RPMRC_FAIL; + } + *generationp = le2ha((unsigned char *)header + XDB_OFFSET_GENERATION); + *slotnpagesp = le2ha((unsigned char *)header + XDB_OFFSET_SLOTNPAGES); + *pagesizep = le2ha((unsigned char *)header + XDB_OFFSET_PAGESIZE); + *usergenerationp = le2ha((unsigned char *)header + XDB_OFFSET_USERGENERATION); + if (!*slotnpagesp || !*pagesizep) + return RPMRC_FAIL; + return RPMRC_OK; +} + static int rpmxdbReadHeader(rpmxdb xdb) { struct xdb_slot *slot; - unsigned int header[XDB_HEADER_SIZE / sizeof(unsigned int)]; - unsigned int slotnpages, pagesize, generation, usergeneration, version; + unsigned int slotnpages, pagesize, generation, usergeneration; unsigned int page, *lastfreep; unsigned char *pageptr; struct xdb_slot *slots, **usedslots, *lastslot; @@ -246,23 +268,9 @@ static int rpmxdbReadHeader(rpmxdb xdb) if (fstat(xdb->fd, &stb)) { return RPMRC_FAIL; } - if (pread(xdb->fd, header, sizeof(header), 0) != sizeof(header)) { - return RPMRC_FAIL; - } - if (le2ha((unsigned char *)header + XDB_OFFSET_MAGIC) != XDB_MAGIC) + if (rpmxdbReadHeaderRaw(xdb, &generation, &slotnpages, &pagesize, &usergeneration)) return RPMRC_FAIL; - version = le2ha((unsigned char *)header + XDB_OFFSET_VERSION); - if (version != XDB_VERSION) { - rpmlog(RPMLOG_ERR, _("rpmxdb: Version mismatch. Expected version: %u. " - "Found version: %u\n"), XDB_VERSION, version); - return RPMRC_FAIL; - } - - generation = le2ha((unsigned char *)header + XDB_OFFSET_GENERATION); - slotnpages = le2ha((unsigned char *)header + XDB_OFFSET_SLOTNPAGES); - pagesize = le2ha((unsigned char *)header + XDB_OFFSET_PAGESIZE); - usergeneration = le2ha((unsigned char *)header + XDB_OFFSET_USERGENERATION); - if (!slotnpages || !pagesize || stb.st_size % pagesize != 0) + if (stb.st_size % pagesize != 0) return RPMRC_FAIL; xdb->pagesize = pagesize; xdb->mapflags = xdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE; @@ -923,6 +931,43 @@ int rpmxdbDelBlob(rpmxdb xdb, unsigned int id) return RPMRC_OK; } +int rpmxdbDelAllBlobs(rpmxdb xdb) +{ + unsigned int slotnpages, pagesize, generation, usergeneration; + if (rpmxdbLockOnly(xdb, 1)) + return RPMRC_FAIL; + /* unmap all blobs */ + if (xdb->slots) { + int i; + struct xdb_slot *slot; + for (i = 1, slot = xdb->slots + i; i < xdb->nslots; i++, slot++) { + if (slot->startpage && slot->mapped) { + unmapslot(xdb, slot); + slot->mapcallback(xdb, slot->mapcallbackdata, 0, 0); + } + } + free(xdb->slots); + xdb->slots = 0; + } + if (xdb->mapped) + unmapheader(xdb); + if (rpmxdbReadHeaderRaw(xdb, &generation, &slotnpages, &pagesize, &usergeneration)) { + rpmxdbUnlock(xdb, 1); + return RPMRC_FAIL; + } + xdb->generation = generation + 1; + xdb->slotnpages = 1; + xdb->pagesize = pagesize; + xdb->usergeneration = usergeneration; + if (rpmxdbWriteEmptySlotpage(xdb, 0)) { + rpmxdbUnlock(xdb, 1); + return RPMRC_FAIL; + } + ftruncate(xdb->fd, xdb->pagesize); + rpmxdbUnlock(xdb, 1); + return RPMRC_OK; +} + int rpmxdbResizeBlob(rpmxdb xdb, unsigned int id, size_t newsize) { struct xdb_slot *slot; diff --git a/lib/backend/ndb/rpmxdb.h b/lib/backend/ndb/rpmxdb.h index 88a3e617a6..ddf02c1bc1 100644 --- a/lib/backend/ndb/rpmxdb.h +++ b/lib/backend/ndb/rpmxdb.h @@ -14,6 +14,7 @@ int rpmxdbUnlock(rpmxdb xdb, int excl); int rpmxdbLookupBlob(rpmxdb xdb, unsigned int *idp, unsigned int blobtag, unsigned int subtag, int flags); int rpmxdbDelBlob(rpmxdb xdb, unsigned int id) ; +int rpmxdbDelAllBlobs(rpmxdb xdb); int rpmxdbMapBlob(rpmxdb xdb, unsigned int id, int flags, void (*mapcallback)(rpmxdb xdb, void *data, void *newaddr, size_t newsize), void *mapcallbackdata); int rpmxdbUnmapBlob(rpmxdb xdb, unsigned int id); From 7d7f889c2f6f880810898540b710a88f8b817036 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 10 Jan 2020 15:51:38 +0100 Subject: [PATCH 3/4] ndb: implement index regeneration if the index is out of sync We compare the user generation stored in the index database with the generation count from the package database. In case there is a mismatch, we delete all the index databases and rely on the already existing missing index creation code to rebuild the database. --- lib/backend/ndb/glue.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/backend/ndb/glue.c b/lib/backend/ndb/glue.c index 841c2fe421..90c10f8894 100644 --- a/lib/backend/ndb/glue.c +++ b/lib/backend/ndb/glue.c @@ -80,6 +80,31 @@ static int ndb_Close(dbiIndex dbi, unsigned int flags) return 0; } +static void ndb_CheckIndexSync(rpmpkgdb pkgdb, rpmxdb xdb) +{ + unsigned int generation, xdb_generation; + if (!pkgdb || !xdb) + return; + if (rpmpkgLock(pkgdb, 0)) + return; + if (rpmpkgGeneration(pkgdb, &generation)) { + rpmpkgUnlock(pkgdb, 0); + return; + } + if (!rpmxdbGetUserGeneration(xdb, &xdb_generation) && generation == xdb_generation) { + rpmpkgUnlock(pkgdb, 0); + return; + } + rpmpkgUnlock(pkgdb, 0); + /* index corrupt or with different generation */ + if (rpmxdbIsRdonly(xdb)) { + rpmlog(RPMLOG_WARNING, _("Detected outdated index databases\n")); + } else { + rpmlog(RPMLOG_WARNING, _("Rebuilding outdated index databases\n")); + rpmxdbDelAllBlobs(xdb); + } +} + static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) { const char *dbhome = rpmdbHome(rdb); @@ -130,6 +155,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) } if (!ndbenv->xdb) { char *path = rstrscat(NULL, dbhome, "/Index.db", NULL); + int created = 0; rpmlog(RPMLOG_DEBUG, "opening db index %s mode=0x%x\n", path, rdb->db_mode); /* Open indexes readwrite if possible */ @@ -144,6 +170,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) } else if (rc && errno == ENOENT) { ioflags = O_CREAT|O_RDWR; rc = rpmxdbOpen(&ndbenv->xdb, rdb->db_pkgs->dbi_db, path, ioflags, 0666); + created = 1; } if (rc) { perror("rpmxdbOpen"); @@ -153,6 +180,8 @@ static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) } free(path); rpmxdbSetFsync(ndbenv->xdb, ndbenv->dofsync); + if (!created) + ndb_CheckIndexSync(ndbenv->pkgdb, ndbenv->xdb); } if (rpmxdbLookupBlob(ndbenv->xdb, &id, rpmtag, 0, 0) == RPMRC_NOTFOUND) { dbi->dbi_flags |= DBI_CREATED; From 24192af913e549f852f52567d59e435ff46f3c51 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 10 Jan 2020 15:56:59 +0100 Subject: [PATCH 4/4] Remove the experimental status from the ndb database --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 7a90b2f4c0..2801b136b0 100644 --- a/configure.ac +++ b/configure.ac @@ -581,7 +581,7 @@ fi #================= # Process --enable-ndb -AC_ARG_ENABLE([ndb], [AS_HELP_STRING([--enable-ndb (EXPERIMENTAL)],[enable the new rpm database format])], +AC_ARG_ENABLE([ndb], [AS_HELP_STRING([--enable-ndb],[enable the new rpm database format])], [case "$enable_ndb" in yes|no) ;; *) AC_MSG_ERROR([invalid argument to --enable-ndb])