Skip to content

Commit 596f4df

Browse files
tobiutobiu
committed
feat(devindex): Implement Star-Based Opt-Out Service (#9230)
1 parent 419fd00 commit 596f4df

6 files changed

Lines changed: 205 additions & 2 deletions

File tree

‎.github/workflows/devindex-pipeline.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ jobs:
2727
- name: Install dependencies
2828
run: npm ci
2929

30+
- name: Run DevIndex Opt-Out
31+
env:
32+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
33+
run: npm run devindex:optout
34+
3035
- name: Run DevIndex Spider (3x Loop)
3136
env:
3237
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎apps/devindex/services/Manager.mjs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import Storage from './Storage.mjs';
66
import Updater from './Updater.mjs';
77
import Spider from './Spider.mjs';
88
import Cleanup from './Cleanup.mjs';
9+
import OptOut from './OptOut.mjs';
910

1011
/**
1112
* @summary DevIndex Backend Orchestrator & CLI Entry Point.
@@ -115,6 +116,13 @@ class Manager extends Base {
115116
await Cleanup.run();
116117
});
117118

119+
program
120+
.command('optout')
121+
.description('Process star-based opt-outs')
122+
.action(async () => {
123+
await OptOut.run();
124+
});
125+
118126
// Initialize Services
119127
await Storage.initAsync();
120128

‎apps/devindex/services/OptOut.mjs‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import Base from '../../../src/core/Base.mjs';
2+
import config from './config.mjs';
3+
import GitHub from './GitHub.mjs';
4+
import Storage from './Storage.mjs';
5+
6+
/**
7+
* @summary Opt-Out Service for DevIndex.
8+
*
9+
* Checks the stargazers of the `neomjs/devindex-opt-out` repository
10+
* to automatically blacklist users and remove them from the index.
11+
*
12+
* @class DevIndex.services.OptOut
13+
* @extends Neo.core.Base
14+
* @singleton
15+
*/
16+
class OptOut extends Base {
17+
static config = {
18+
/**
19+
* @member {String} className='DevIndex.services.OptOut'
20+
* @protected
21+
*/
22+
className: 'DevIndex.services.OptOut',
23+
/**
24+
* @member {Boolean} singleton=true
25+
* @protected
26+
*/
27+
singleton: true,
28+
/**
29+
* @member {String} optOutRepoOwner='neomjs'
30+
*/
31+
optOutRepoOwner: 'neomjs',
32+
/**
33+
* @member {String} optOutRepoName='devindex-opt-out'
34+
*/
35+
optOutRepoName: 'devindex-opt-out'
36+
}
37+
38+
async run() {
39+
console.log('[OptOut] Checking for new opt-out requests...');
40+
const syncState = await Storage.getOptOutSync();
41+
const lastCheck = syncState.lastCheck;
42+
let newLastCheck = lastCheck;
43+
44+
let hasNextPage = true;
45+
let cursor = null;
46+
let optedOutLogins = [];
47+
48+
while (hasNextPage) {
49+
const query = `
50+
query($owner: String!, $name: String!, $cursor: String) {
51+
repository(owner: $owner, name: $name) {
52+
stargazers(first: 100, orderBy: {field: STARRED_AT, direction: DESC}, after: $cursor) {
53+
pageInfo {
54+
hasNextPage
55+
endCursor
56+
}
57+
edges {
58+
starredAt
59+
node {
60+
login
61+
}
62+
}
63+
}
64+
}
65+
}`;
66+
67+
const variables = {
68+
owner: this.optOutRepoOwner,
69+
name: this.optOutRepoName,
70+
cursor: cursor
71+
};
72+
73+
let data;
74+
try {
75+
data = await GitHub.query(query, variables, 3, 'OptOut');
76+
} catch (err) {
77+
// If repo doesn't exist yet, just warn and exit gracefully
78+
if (err.message.includes('NOT_FOUND') || err.message.includes('Could not resolve')) {
79+
console.warn(`[OptOut] Opt-out repository ${this.optOutRepoOwner}/${this.optOutRepoName} not found. Skipping.`);
80+
return;
81+
}
82+
throw err;
83+
}
84+
85+
const stargazers = data?.repository?.stargazers;
86+
if (!stargazers) break;
87+
88+
const edges = stargazers.edges || [];
89+
let stopFetching = false;
90+
91+
for (const edge of edges) {
92+
const starredAt = edge.starredAt;
93+
const login = edge.node.login;
94+
95+
// Stop if we reached stars we've already processed
96+
if (lastCheck && starredAt <= lastCheck) {
97+
stopFetching = true;
98+
break;
99+
}
100+
101+
// Keep track of the newest timestamp to save later
102+
if (!newLastCheck || starredAt > newLastCheck) {
103+
newLastCheck = starredAt;
104+
}
105+
106+
optedOutLogins.push(login);
107+
}
108+
109+
if (stopFetching) {
110+
break;
111+
}
112+
113+
hasNextPage = stargazers.pageInfo.hasNextPage;
114+
cursor = stargazers.pageInfo.endCursor;
115+
}
116+
117+
if (optedOutLogins.length > 0) {
118+
console.log(`[OptOut] Found ${optedOutLogins.length} new opt-out requests.`);
119+
120+
// 1. Add to blacklist
121+
await Storage.addToBlacklist(optedOutLogins);
122+
123+
// 2. Remove from rich data (users.jsonl)
124+
await Storage.deleteUsers(optedOutLogins);
125+
126+
// 3. Remove from tracker
127+
const trackerUpdates = optedOutLogins.map(login => ({ login, delete: true }));
128+
await Storage.updateTracker(trackerUpdates);
129+
130+
// 4. Remove from failed list (Penalty Box)
131+
await Storage.updateFailed(optedOutLogins, false);
132+
133+
// Update sync state
134+
await Storage.saveOptOutSync({ lastCheck: newLastCheck });
135+
console.log(`[OptOut] Processed opt-outs and updated sync state to ${newLastCheck}.`);
136+
} else {
137+
console.log('[OptOut] No new opt-out requests found.');
138+
}
139+
}
140+
}
141+
142+
export default Neo.setupClass(OptOut);

‎apps/devindex/services/Storage.mjs‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ class Storage extends Base {
6464
{ path: config.paths.blacklist, default: [] },
6565
{ path: config.paths.whitelist, default: [] },
6666
{ path: config.paths.failed, default: {} },
67-
{ path: config.paths.threshold, default: { tc: config.github.minTotalContributions } }
67+
{ path: config.paths.threshold, default: { tc: config.github.minTotalContributions } },
68+
{ path: config.paths.optoutSync, default: { lastCheck: null } }
6869
];
6970

7071
for (const file of files) {
@@ -86,6 +87,46 @@ class Storage extends Base {
8687
return new Set(list.map(item => item.toLowerCase()));
8788
}
8889

90+
/**
91+
* Adds users to the blacklist.
92+
* @param {Array<String>} logins
93+
* @returns {Promise<void>}
94+
*/
95+
async addToBlacklist(logins) {
96+
const current = await this.readJson(config.paths.blacklist, []);
97+
const currentSet = new Set(current.map(item => item.toLowerCase()));
98+
let changed = false;
99+
100+
for (const login of logins) {
101+
if (!currentSet.has(login.toLowerCase())) {
102+
current.push(login);
103+
currentSet.add(login.toLowerCase());
104+
changed = true;
105+
}
106+
}
107+
108+
if (changed) {
109+
await this.writeJson(config.paths.blacklist, current);
110+
}
111+
}
112+
113+
/**
114+
* Reads the opt-out sync state.
115+
* @returns {Promise<Object>}
116+
*/
117+
async getOptOutSync() {
118+
return this.readJson(config.paths.optoutSync, { lastCheck: null });
119+
}
120+
121+
/**
122+
* Saves the opt-out sync state.
123+
* @param {Object} data
124+
* @returns {Promise<void>}
125+
*/
126+
async saveOptOutSync(data) {
127+
await this.writeJson(config.paths.optoutSync, data);
128+
}
129+
89130
/**
90131
* Reads the whitelist.
91132
* @returns {Promise<Set<String>>} Set of whitelisted logins.

‎apps/devindex/services/config.mjs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,13 @@ const defaultConfig = {
135135
* Stores the minimum total contributions required to enter the index.
136136
* @type {string}
137137
*/
138-
threshold: path.resolve(projectRoot, 'apps/devindex/resources/threshold.json')
138+
threshold: path.resolve(projectRoot, 'apps/devindex/resources/threshold.json'),
139+
140+
/**
141+
* State tracking for the Opt-Out service (last processed timestamp).
142+
* @type {string}
143+
*/
144+
optoutSync: path.resolve(projectRoot, 'apps/devindex/resources/optout-sync.json')
139145
}
140146
};
141147

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
"create-component" : "node ./buildScripts/create/component.mjs",
4242
"devindex:add" : "node ./apps/devindex/services/cli.mjs add",
4343
"devindex:cleanup" : "node ./apps/devindex/services/cli.mjs cleanup",
44+
"devindex:optout" : "node ./apps/devindex/services/cli.mjs optout",
4445
"devindex:spider" : "node ./apps/devindex/services/cli.mjs spider",
4546
"devindex:update" : "node ./apps/devindex/services/cli.mjs update --limit 500",
4647
"generate-docs-json" : "node ./buildScripts/docs/jsdocx.mjs",

0 commit comments

Comments
 (0)