fix: repair four OOM error-path bugs found by malloc-fault injection

src/db/db_conv.c

@@ -73,6 +73,17 @@ __db_pgin(dbenv, pg, pp, cookie)
 	int is_hmac, ret;
 	u_int8_t *chksum;
 
+	/*
+	 * The pgin/pgout cookie carries the DB_PGINFO the swap needs.  If a
+	 * file was registered for page-in but its cookie was never set (e.g.
+	 * an allocation failed while opening the file), __memp_get_pgcookie
+	 * hands back an empty DBT.  A page swap cannot proceed without the
+	 * page size and flags, so treat a missing/short cookie as an error
+	 * rather than dereferencing past it.
+	 */
+	if (cookie == NULL || cookie->data == NULL ||
+	    cookie->size < sizeof(DB_PGINFO))
+		return (EINVAL);
 	pginfo = (DB_PGINFO *)cookie->data;
 	env = dbenv->env;
 	pagep = (PAGE *)pp;
@@ -225,6 +236,10 @@ __db_pgout(dbenv, pg, pp, cookie)
 	PAGE *pagep;
 	int ret;
 
+	/* See __db_pgin: a page swap needs a valid DB_PGINFO cookie. */
+	if (cookie == NULL || cookie->data == NULL ||
+	    cookie->size < sizeof(DB_PGINFO))
+		return (EINVAL);
 	pginfo = (DB_PGINFO *)cookie->data;
 	env = dbenv->env;
 	pagep = (PAGE *)pp;

src/env/env_method.c

@@ -116,25 +116,33 @@ void
 __db_env_destroy(dbenv)
 	DB_ENV *dbenv;
 {
-	__lock_env_destroy(dbenv);
-	__log_env_destroy(dbenv);
-	__memp_env_destroy(dbenv);
+	/*
+	 * The subsystem destructors and the ENV teardown below all dereference
+	 * dbenv->env.  On the early-failure path in __env_create (the ENV calloc
+	 * itself failed) dbenv->env is still NULL, so guard the whole ENV cleanup
+	 * -- there is nothing to tear down and dereferencing it would crash.
+	 */
+	if (dbenv->env != NULL) {
+		__lock_env_destroy(dbenv);
+		__log_env_destroy(dbenv);
+		__memp_env_destroy(dbenv);
 #ifdef HAVE_REPLICATION
-	__rep_env_destroy(dbenv);
+		__rep_env_destroy(dbenv);
 #endif
-	__txn_env_destroy(dbenv);
+		__txn_env_destroy(dbenv);
 
-	/*
-	 * Discard the underlying ENV structure.
-	 *
-	 * XXX
-	 * This is wrong, but can't be fixed until we finish the work of
-	 * splitting up the DB_ENV and ENV structures so that we don't
-	 * touch anything in the ENV as part of the above calls to subsystem
-	 * DB_ENV cleanup routines.
-	 */
-	memset(dbenv->env, CLEAR_BYTE, sizeof(ENV));
-	__os_free(NULL, dbenv->env);
+		/*
+		 * Discard the underlying ENV structure.
+		 *
+		 * XXX
+		 * This is wrong, but can't be fixed until we finish the work of
+		 * splitting up the DB_ENV and ENV structures so that we don't
+		 * touch anything in the ENV as part of the above calls to subsystem
+		 * DB_ENV cleanup routines.
+		 */
+		memset(dbenv->env, CLEAR_BYTE, sizeof(ENV));
+		__os_free(NULL, dbenv->env);
+	}
 
 	memset(dbenv, CLEAR_BYTE, sizeof(DB_ENV));
 	__os_free(NULL, dbenv);

src/lock/lock_id.c

@@ -338,20 +338,30 @@ __lock_getlocker_int(lt, locker, create, retp)
 			 * allocate as much as possible.
 			 */
 			F_SET(&lt->reginfo, REGION_TRACKED);
+			/*
+			 * Try to allocate nlockers entries; on failure halve the
+			 * request and retry, down to zero.  The halving assignment
+			 * (nlockers >>= 1) is essential: without it the loop either
+			 * spins on the same failing size or breaks with sh_locker
+			 * still NULL while nlockers is non-zero -- the insert loop
+			 * below would then dereference a NULL sh_locker.  When the
+			 * request cannot be satisfied at all, nlockers reaches 0 and
+			 * we take the __lock_nomem path before touching sh_locker.
+			 */
 			while (__env_alloc(&lt->reginfo, nlockers *
 			    sizeof(struct __db_locker), &sh_locker) != 0)
-				if ((nlockers >> 1) == 0)
+				if ((nlockers >>= 1) == 0)
 					break;
 			F_CLR(&lt->reginfo, REGION_TRACKED);
 			LOCK_REGION_UNLOCK(lt->env);
 			LOCK_LOCKERS(env, region);
+			if (nlockers == 0)
+				return (__lock_nomem(env, "locker entries"));
 			for (i = 0; i < nlockers; i++) {
 				SH_TAILQ_INSERT_HEAD(&region->free_lockers,
 				    sh_locker, links, __db_locker);
 				sh_locker++;
 			}
-			if (nlockers == 0)
-				return (__lock_nomem(env, "locker entries"));
 			region->stat.st_lockers += nlockers;
 			sh_locker = SH_TAILQ_FIRST(
 			    &region->free_lockers, __db_locker);

src/txn/txn.c

@@ -1760,6 +1760,20 @@ __txn_end(txn, is_commit)
 			return (__env_panic(env, ret));
 	}
 
+	/*
+	 * Free the locker BEFORE the transaction detail (td): __lock_freelocker
+	 * reads td->si_ref (via the locker's td_off) to decide whether committed
+	 * SIREAD markers still reference the locker.  The detail is freed below
+	 * under TXN_SYSTEM_LOCK, so the locker must be released here, while td is
+	 * still alive -- otherwise __lock_freelocker_int dereferences a freed td
+	 * (a use-after-free on the OOM/abort cleanup path).  The locker lives in
+	 * its own lock domain (LOCK_LOCKERS), independent of TXN_SYSTEM_LOCK, so
+	 * releasing it here does not change lock ordering.
+	 */
+	if (LOCKING_ON(env) && (ret =
+	    __lock_freelocker(env->lk_handle, txn->locker)) != 0)
+		return (__env_panic(env, ret));
+
 	TXN_SYSTEM_LOCK(env);
 	td->status = is_commit ? TXN_COMMITTED : TXN_ABORTED;
 	SH_TAILQ_REMOVE(&region->active_txn, td, links, __txn_detail);
@@ -1846,13 +1860,6 @@ __txn_end(txn, is_commit)
 
 	TXN_SYSTEM_UNLOCK(env);
 
-	/*
-	 * The transaction cannot get more locks, remove its locker info,
-	 * if any.
-	 */
-	if (LOCKING_ON(env) && (ret =
-	    __lock_freelocker(env->lk_handle, txn->locker)) != 0)
-		return (__env_panic(env, ret));
 	if (txn->parent != NULL)
 		TAILQ_REMOVE(&txn->parent->kids, txn, klinks);