Skip to content

Commit 67375c0

Browse files
SysSn13SysSn13
committed
added new job for updating users' data periodically and updated the prediction flow. No longer storing user contest history as it's time consuming.
1 parent 581ecbb commit 67375c0

File tree

16 files changed

+495
-420
lines changed

16 files changed

+495
-420
lines changed

‎background.js

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,25 @@ const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
1414
serverAdapter.setBasePath("/bull-board");
1515

1616
const initScheduler = async () => {
17-
await jobScheduler.add({});
18-
await jobScheduler.add({}, { repeat: { cron: "0 0 * * *" } });
17+
await jobScheduler.add("contestScheduler", {});
18+
await jobScheduler.add("updateUserDataScheduler", {
19+
rateLimit: 3,
20+
limit: 1000,
21+
});
22+
23+
// repeat contestScheduler every day at midnight
24+
await jobScheduler.add(
25+
"contestScheduler",
26+
{},
27+
{ repeat: { cron: "0 0 * * *" } }
28+
);
29+
30+
// Repeat updateUserDataScheduler every 4 hours
31+
await jobScheduler.add(
32+
"updateUserDataScheduler",
33+
{ rateLimit: 3, limit: 1000 },
34+
{ repeat: { cron: "0 */4 * * *" } }
35+
);
1936
};
2037
initScheduler();
2138

‎helpers.js

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
exports.getUserId = (username, dataRegion = "US") => {
2+
return dataRegion + "/" + username.trim().toLowerCase();
3+
};
4+
5+
exports.convertDateYYYYMMDD = (date) => {
6+
var yyyy = date.getFullYear().toString();
7+
var mm = (date.getMonth() + 1).toString();
8+
var dd = date.getDate().toString();
9+
10+
var mmChars = mm.split("");
11+
var ddChars = dd.split("");
12+
13+
return (
14+
yyyy +
15+
"-" +
16+
(mmChars[1] ? mm : "0" + mmChars[0]) +
17+
"-" +
18+
(ddChars[1] ? dd : "0" + ddChars[0])
19+
);
20+
};
21+
22+
exports.IsLatestContest = (time) => {
23+
return Date.now() - time <= 2 * 24 * 60 * 60 * 1000;
24+
};
25+
26+
exports.getRemainingTime = (time) => {
27+
return Math.max(0, Math.ceil(time - Date.now()));
28+
};
29+
30+
exports.isNumeric = (value) => {
31+
return /^\d+$/.test(value);
32+
};

‎models/contest.js

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const rankingSchema = new Schema({
2222
});
2323
const ContestRankingsSchema = new Schema({
2424
_id: String,
25+
title: String,
2526
startTime: Date,
2627
endTime: Date,
2728
contest_id: Number,
@@ -30,6 +31,10 @@ const ContestRankingsSchema = new Schema({
3031
type: Boolean,
3132
default: false,
3233
},
34+
users_fetched: {
35+
type: Boolean,
36+
default: false,
37+
},
3338
ratings_predicted: {
3439
type: Boolean,
3540
default: false,

‎models/users.js renamed to ‎models/user.js

File renamed without changes.

‎services/contests.js

Lines changed: 25 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,15 @@
11
const fetch = require("node-fetch");
22
const Contest = require("../models/contest");
3+
const { IsLatestContest } = require("../helpers");
34

4-
// async function pushRankingsToContest(
5-
// contest_id,
6-
// rankings,
7-
// pagesCnt,
8-
// islastPage
9-
// ) {
10-
// try {
11-
// await Contest.findOneAndUpdate(
12-
// {
13-
// _id: contest_id,
14-
// },
15-
// {
16-
// $push: {
17-
// rankings: rankings,
18-
// },
19-
// $inc: {
20-
// pages_fetched: pagesCnt,
21-
// },
22-
// $set: {
23-
// rankings_fetched: islastPage,
24-
// },
25-
// }
26-
// );
27-
28-
// return null;
29-
// } catch (err) {
30-
// console.log(err);
31-
// return err;
32-
// }
33-
// }
345
const fetchContestRankings = async function (contestSlug) {
356
try {
36-
let contest = await Contest.findById(contestSlug, { rankings: 0 });
7+
let contest = await Contest.findById(contestSlug);
378
if (!contest) {
38-
return Error(`Contest ${contestSlug} not found in the db`);
9+
return [null, Error(`Contest ${contestSlug} not found in the db`)];
10+
}
11+
if (contest.rankings_fetched) {
12+
return [contest, null];
3913
}
4014

4115
console.log(`fetching ${contestSlug} ...`);
@@ -46,16 +20,23 @@ const fetchContestRankings = async function (contestSlug) {
4620
let pages = Math.floor(resp.user_num / 25);
4721
let all_rankings = [];
4822
let failed = [];
23+
let lastPage = Math.MAX_SAFE_INTEGER;
4924
const fetchPageRankings = async (
5025
pageNo,
5126
retries,
5227
throwError = false
5328
) => {
29+
if (pageNo > lastPage) {
30+
return;
31+
}
5432
console.log(`Fetching rankings (${contestSlug}): page: ${pageNo}`);
5533
try {
5634
let res = await fetch(
5735
`https://leetcode.com/contest/api/ranking/${contestSlug}/?pagination=${pageNo}&region=global`
5836
);
37+
if (res.status !== 200) {
38+
throw new Error(res.statusText);
39+
}
5940
res = await res.json();
6041
rankings = res.total_rank
6142
.filter(
@@ -86,15 +67,18 @@ const fetchContestRankings = async function (contestSlug) {
8667
ranking["_id"] = username;
8768
return ranking;
8869
});
70+
if (rankings.length < 25) {
71+
lastPage = Math.min(lastPage, pageNo);
72+
}
8973
all_rankings = all_rankings.concat(rankings);
9074
console.log(
91-
`Fetched rankings (${contestSlug}): page: ${pageNo}`
75+
`Fetched rankings (${contestSlug} page: ${pageNo})`
9276
);
9377
} catch (err) {
9478
console.log(
95-
`Failed to fetch rankings (${contestSlug}, page: ${pageNo})`
79+
`Failed to fetch rankings (${contestSlug} page: ${pageNo})`,
80+
err.message
9681
);
97-
console.log(err);
9882
if (retries > 0) {
9983
await fetchPageRankings(pageNo, retries - 1);
10084
} else if (throwError) {
@@ -121,11 +105,12 @@ const fetchContestRankings = async function (contestSlug) {
121105
all_rankings.sort((a, b) => (a.rank > b.rank ? 1 : -1));
122106
contest.rankings = all_rankings;
123107
contest.rankings_fetched = true;
124-
contest.user_num = resp.user_num;
125-
await contest.save();
108+
contest.user_num = all_rankings.length;
109+
await contest.update();
126110
console.log(`Updated Rankings in ${contestSlug}.`);
111+
return [contest, null];
127112
} catch (err) {
128-
return err;
113+
return [null, err];
129114
}
130115
};
131116
const fetchContestsMetaData = async () => {
@@ -163,14 +148,10 @@ const fetchContestsMetaData = async () => {
163148
});
164149
res = await res.json();
165150

166-
const withinAWeek = (endTime) => {
167-
return Date.now() - endTime <= 7 * 24 * 60 * 60 * 1000;
168-
};
169-
170151
for (let i = 0; i < res.data.allContests.length; i++) {
171152
let contest = res.data.allContests[i];
172153
const endTime = contest.startTime * 1000 + contest.duration * 1000;
173-
if (!withinAWeek(endTime)) {
154+
if (!IsLatestContest(endTime)) {
174155
continue;
175156
}
176157
let contestExists = await Contest.exists({
@@ -179,13 +160,12 @@ const fetchContestsMetaData = async () => {
179160
if (contestExists) {
180161
continue;
181162
}
182-
console.log(contest.titleSlug);
183163
let newContest = new Contest({
184164
_id: contest.titleSlug,
165+
title: contest.title,
185166
startTime: contest.startTime * 1000,
186167
endTime: endTime,
187168
lastUpdated: Date.now(),
188-
num_user: contest.num_user,
189169
});
190170
await newContest.save();
191171
console.log(`created new contest: ${contest.titleSlug}`);
@@ -197,25 +177,7 @@ const fetchContestsMetaData = async () => {
197177
return null;
198178
}
199179
};
200-
const updateContestRankings = async function (contestSlug) {
201-
try {
202-
let contest = await Contest.findById(contestSlug, {
203-
rankings: 0,
204-
});
205-
206-
if (!contest) {
207-
return Error(`contest ${contestSlug} not found in the db.`);
208-
} else if (!contest.rankings_fetched) {
209-
let err = await fetchContestRankings(contestSlug);
210-
return err;
211-
}
212-
return null;
213-
} catch (err) {
214-
return err;
215-
}
216-
};
217180

218181
// exports
219182
exports.fetchContestsMetaData = fetchContestsMetaData;
220-
exports.updateContestRankings = updateContestRankings;
221183
exports.fetchContestRankings = fetchContestRankings;

‎services/helpers.js

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎services/job-queues/contestPredictionQueue.js

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
const Queue = require("bull");
22
const { predict } = require("../predict");
33
const opts = require("../redis");
4+
const { updateUsers } = require("../users");
45

56
opts.lockDuration = 30 * 60 * 1000; // 30 minutes
67

78
const predictQueue = new Queue("Predictions", opts);
8-
predictQueue.process(async (job, done) => {
9+
predictQueue.process("predictRatings", async (job, done) => {
910
console.log("Processing job: ", job.id);
1011
const err = await predict(job);
1112
if (err) {
@@ -15,8 +16,49 @@ predictQueue.process(async (job, done) => {
1516
done();
1617
});
1718

19+
predictQueue.process("updateUserData", async (job, done) => {
20+
try {
21+
const predictRatingJobScheduled = await checkPredictRatingJobs();
22+
if (predictRatingJobScheduled) {
23+
done(
24+
new Error(
25+
"PredictRating job is scheduled. Cannot process this job right now."
26+
)
27+
);
28+
return;
29+
}
30+
console.log("Processing job: ", job.id);
31+
const err = await updateUsers(job);
32+
if (err) {
33+
console.error(err);
34+
done(err);
35+
} else {
36+
done();
37+
}
38+
} catch (err) {
39+
done(err);
40+
}
41+
});
42+
43+
// funtion to check if any predictRating job is scheduled
44+
const checkPredictRatingJobs = async () => {
45+
let jobs = await predictQueue.getJobs([
46+
"active",
47+
"waiting",
48+
"delayed",
49+
"failed",
50+
]);
51+
jobs = jobs.filter((job) => job.name === "predictRatings");
52+
let date = new Date();
53+
date.setDate(date.getDate() + 30);
54+
for (job of jobs) {
55+
date = Math.min(date, new Date(job.opts.timestamp + job.opts.delay));
56+
}
57+
return date - Date.now() <= 1 * 24 * 60 * 60 * 100;
58+
};
59+
1860
predictQueue.on("completed", (job, result) => {
19-
console.log(`Prediction Job: ${job.id} completed.`);
61+
console.log(`${job.name} Job: ${job.id} completed.`);
2062
});
2163

2264
module.exports = predictQueue;

0 commit comments

Comments
 (0)