Implement a key-only rpmdb index iterator

The regular index iterator grabs the associated data too, which we don't always need. The data associated with indexes is relatively lightweight, but as with everything, it adds up if in the millions scale.
rpm-software-management · Dec 5, 2019 · 16d9d3a · 16d9d3a
1 parent 61ea5a8
commit 16d9d3a
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 20 deletions.
diff --git a/lib/backend/db3.c b/lib/backend/db3.c
@@ -1008,7 +1008,7 @@ static rpmRC db3_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
 			  dbiIndexSet *set, int searchType)
 {
     rpmRC rc = RPMRC_FAIL; /* assume failure */
-    if (dbi != NULL && dbc != NULL && set != NULL) {
+    if (dbi != NULL && dbc != NULL) {
 	int cflags = DB_NEXT;
 	int dbrc;
 	DBT data, key;
@@ -1033,12 +1033,14 @@ static rpmRC db3_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
 	    if (searchType == DBC_PREFIX_SEARCH &&
 		    (key.size < keylen || memcmp(key.data, keyp, keylen) != 0))
 		break;
-	    dbt2set(dbi, &data, &newset);
-	    if (*set == NULL) {
-		*set = newset;
-	    } else {
-		dbiIndexSetAppendSet(*set, newset, 0);
-		dbiIndexSetFree(newset);
+	    if (set) {
+		dbt2set(dbi, &data, &newset);
+		if (*set == NULL) {
+		    *set = newset;
+		} else {
+		    dbiIndexSetAppendSet(*set, newset, 0);
+		    dbiIndexSetFree(newset);
+		}
 	    }
 	    if (searchType != DBC_PREFIX_SEARCH)
 		break;
@@ -1048,7 +1050,7 @@ static rpmRC db3_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
 	}
 
 	/* fixup result status for prefix search */
-	if (searchType == DBC_PREFIX_SEARCH) {
+	if (searchType == DBC_PREFIX_SEARCH && set) {
 	    if (dbrc == DB_NOTFOUND && *set != NULL && (*set)->count > 0)
 		dbrc = 0;
 	    else if (dbrc == 0 && (*set == NULL || (*set)->count == 0))

diff --git a/lib/backend/lmdb.c b/lib/backend/lmdb.c
@@ -574,7 +574,7 @@ static rpmRC lmdb_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
 			  dbiIndexSet *set, int searchType)
 {
     rpmRC rc = RPMRC_FAIL; /* assume failure */
-    if (dbi != NULL && dbc != NULL && set != NULL) {
+    if (dbi != NULL && dbc != NULL) {
 	int cflags = MDB_NEXT;
 	int dbrc;
 	MDB_val key = { 0, NULL };
@@ -598,12 +598,14 @@ static rpmRC lmdb_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
 	    if (searchType == DBC_PREFIX_SEARCH &&
 		    (key.mv_size < keylen || memcmp(key.mv_data, keyp, keylen) != 0))
 		break;
-	    dbt2set(dbi, &data, &newset);
-	    if (*set == NULL) {
-		*set = newset;
-	    } else {
-		dbiIndexSetAppendSet(*set, newset, 0);
-		dbiIndexSetFree(newset);
+	    if (set) {
+		dbt2set(dbi, &data, &newset);
+		if (*set == NULL) {
+		    *set = newset;
+		} else {
+		    dbiIndexSetAppendSet(*set, newset, 0);
+		    dbiIndexSetFree(newset);
+		}
 	    }
 	    if (searchType != DBC_PREFIX_SEARCH)
 		break;
@@ -613,7 +615,7 @@ static rpmRC lmdb_idxdbGet(dbiIndex dbi, dbiCursor dbc, const char *keyp, size_t
 	}
 
 	/* fixup result status for prefix search */
-	if (searchType == DBC_PREFIX_SEARCH) {
+	if (searchType == DBC_PREFIX_SEARCH && set) {
 	    if (dbrc == MDB_NOTFOUND && *set != NULL && (*set)->count > 0)
 		dbrc = 0;
 	    else if (dbrc == 0 && (*set == NULL || (*set)->count == 0))

diff --git a/lib/backend/ndb/glue.c b/lib/backend/ndb/glue.c
@@ -409,6 +409,14 @@ static rpmRC ndb_idxdbIter(dbiIndex dbi, dbiCursor dbc, dbiIndexSet *set)
 	    break;
 	}
 #endif
+	if (set == NULL) {
+	    dbc->key = k;
+	    dbc->keylen = kl;
+	    dbc->ilist += 2;
+	    rc = RPMRC_OK;
+	    break;
+	}
+
 	pkglist = 0;
 	pkglistn = 0;
 	rc = rpmidxGet(dbc->dbi->dbi_db, k, kl, &pkglist, &pkglistn);

diff --git a/lib/backend/sqlite.c b/lib/backend/sqlite.c
@@ -578,15 +578,17 @@ static rpmRC sqlite_idxdbIter(dbiIndex dbi, dbiCursor dbc, const char *keyp, siz
 	rc = sqlite3_step(dbc->stmt);
 
     if (rc == SQLITE_ROW) {
-	dbiCursor kc = dbiCursorInit(dbi, 0);
 	if (dbc->ctype == SQLITE_TEXT) {
 	    dbc->key = sqlite3_column_text(dbc->stmt, 0);
 	} else {
 	    dbc->key = sqlite3_column_blob(dbc->stmt, 0);
 	}
 	dbc->keylen = sqlite3_column_bytes(dbc->stmt, 0);
-	rc = sqlite_idxdbByKey(dbi, kc, dbc->key, dbc->keylen, set);
-	dbiCursorFree(dbi, kc);
+	if (set) {
+	    dbiCursor kc = dbiCursorInit(dbi, 0);
+	    rc = sqlite_idxdbByKey(dbi, kc, dbc->key, dbc->keylen, set);
+	    dbiCursorFree(dbi, kc);
+	}
 	rc = RPMRC_OK;
     } else if (rc == SQLITE_DONE) {
 	if (searchType == DBC_PREFIX_SEARCH && (*set))

diff --git a/lib/rpmdb.c b/lib/rpmdb.c
@@ -298,6 +298,7 @@ struct rpmdbIndexIterator_s {
     dbiCursor		ii_dbc;
     dbiIndexSet		ii_set;
     unsigned int	*ii_hdrNums;
+    int			skipdata;
 };
 
 static rpmdb rpmdbRock;
@@ -1885,6 +1886,13 @@ rpmdbIndexIterator rpmdbIndexIteratorInit(rpmdb db, rpmDbiTag rpmtag)
     return ii;
 }
 
+rpmdbIndexIterator rpmdbIndexKeyIteratorInit(rpmdb db, rpmDbiTag rpmtag)
+{
+    rpmdbIndexIterator ki = rpmdbIndexIteratorInit(db, rpmtag);
+    ki->skipdata = 1;
+    return ki;
+}
+
 int rpmdbIndexIteratorNext(rpmdbIndexIterator ii, const void ** key, size_t * keylen)
 {
     int rc;
@@ -1899,7 +1907,8 @@ int rpmdbIndexIteratorNext(rpmdbIndexIterator ii, const void ** key, size_t * ke
     /* free old data */
     ii->ii_set = dbiIndexSetFree(ii->ii_set);
 
-    rc = idxdbGet(ii->ii_dbi, ii->ii_dbc, NULL, 0, &ii->ii_set, DBC_NORMAL_SEARCH);
+    rc = idxdbGet(ii->ii_dbi, ii->ii_dbc, NULL, 0,
+		ii->skipdata ? NULL : &ii->ii_set, DBC_NORMAL_SEARCH);
 
     *key = idxdbKey(ii->ii_dbi, ii->ii_dbc, &iikeylen);
     *keylen = iikeylen;

diff --git a/lib/rpmdb.h b/lib/rpmdb.h
@@ -153,6 +153,14 @@ Header rpmdbNextIterator(rpmdbMatchIterator mi);
  */
 rpmdbMatchIterator rpmdbFreeIterator(rpmdbMatchIterator mi);
 
+/** \ingroup rpmdb
+ * Get an iterator for index keys
+ * @param db		rpm database
+ * @param rpmtag	the index to iterate over
+ * @return		the index iterator
+ */
+rpmdbIndexIterator rpmdbIndexKeyIteratorInit(rpmdb db, rpmDbiTag rpmtag);
+
 /** \ingroup rpmdb
  * Get an iterator for an index
  * @param db		rpm database