From 2a5f8ab27edb94afba41aa9db7f8a646667f485b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Soner=20U=C5=9Fakl=C4=B1?=
 <barisusakli@gmail.com>
Date: Tue, 21 Jul 2020 18:19:46 -0400
Subject: [PATCH] feat: dont load all subscribers at once

increase batch to 500
---
 src/user/digest.js | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/user/digest.js b/src/user/digest.js
index 90b85bc0380f..ccf263440606 100644
--- a/src/user/digest.js
+++ b/src/user/digest.js
@@ -29,13 +29,14 @@ Digest.execute = async function (payload) {
 		return;
 	}
 	try {
+		winston.info('[user/jobs] Digest (' + payload.interval + ') scheduling completed. Sending emails; this may take some time...');
 		await Digest.send({
 			interval: payload.interval,
 			subscribers: subscribers,
 		});
-		winston.info('[user/jobs] Digest (' + payload.interval + ') scheduling completed. Sending emails; this may take some time...');
+		winston.info('[user/jobs] Digest (' + payload.interval + ') complete.');
 	} catch (err) {
-		winston.error('[user/jobs] Could not send digests (' + payload.interval + ')', err.stack);
+		winston.error('[user/jobs] Could not send digests (' + payload.interval + ')\n' + err.stack);
 		throw err;
 	}
 };
@@ -81,7 +82,10 @@ Digest.getSubscribers = async function (interval) {
 		});
 		subUids = await user.bans.filterBanned(subUids);
 		subscribers = subscribers.concat(subUids);
-	}, { interval: 1000 });
+	}, {
+		interval: 1000,
+		batch: 500,
+	});
 
 	const results = await plugins.fireHook('filter:digest.subscribers', {
 		interval: interval,
@@ -91,15 +95,13 @@ Digest.getSubscribers = async function (interval) {
 };
 
 Digest.send = async function (data) {
-	var emailsSent = 0;
+	let emailsSent = 0;
 	if (!data || !data.subscribers || !data.subscribers.length) {
 		return emailsSent;
 	}
-	const now = new Date();
-
-	const users = await user.getUsersFields(data.subscribers, ['uid', 'username', 'userslug', 'lastonline']);
 
-	async.eachLimit(users, 100, async function (userObj) {
+	await async.eachLimit(data.subscribers, 100, async function (uid) {
+		const userObj = await user.getUserFields(uid, ['uid', 'username', 'userslug', 'lastonline']);
 		let [notifications, topicsData] = await Promise.all([
 			user.notifications.getUnreadInterval(userObj.uid, data.interval),
 			getTermTopics(data.interval, userObj.uid, 0, 9),
@@ -128,6 +130,7 @@ Digest.send = async function (data) {
 			return topicObj;
 		});
 		emailsSent += 1;
+		const now = Date.now();
 		try {
 			await emailer.send('digest', userObj.uid, {
 				subject: '[[email:digest.subject, ' + (now.getFullYear() + '/' + (now.getMonth() + 1) + '/' + now.getDate()) + ']]',
@@ -145,9 +148,8 @@ Digest.send = async function (data) {
 		if (data.interval !== 'alltime') {
 			await db.sortedSetAdd('digest:delivery', now.getTime(), userObj.uid);
 		}
-	}, function () {
-		winston.info('[user/jobs] Digest (' + data.interval + ') sending completed. ' + emailsSent + ' emails sent.');
 	});
+	winston.info('[user/jobs] Digest (' + data.interval + ') sending completed. ' + emailsSent + ' emails sent.');
 };
 
 Digest.getDeliveryTimes = async (start, stop) => {