-
Notifications
You must be signed in to change notification settings - Fork 27
/
check-sources-updates.js
411 lines (339 loc) · 11.5 KB
/
check-sources-updates.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
/* eslint-disable no-console, camelcase */
const fs = require('fs');
const axios = require('axios');
const { EOL } = require('os');
const {
REMOVED_MARKER,
COMPATIBILITY_TABLE_DATA_PATH,
} = require('./constants');
/* ************************************************************************
*
* Common
*
************************************************************************** */
/**
* Checks if arrays contain the same strings
*
* @param {Array} arr1
* @param {Array} arr2
* @returns {boolean}
*/
const areArraysOfStringsEqual = (arr1, arr2) => {
if (arr1.length !== arr2.length) {
return false;
}
const sorted1 = arr1.sort();
const sorted2 = arr2.sort();
return sorted1.every((value, index) => value === sorted2[index]);
};
/**
* Returns parsed compatibility table
*
* @returns {Object}
*/
const getCompatibilityTable = () => {
const rawData = fs.readFileSync(COMPATIBILITY_TABLE_DATA_PATH);
const parsed = JSON.parse(rawData);
return parsed;
};
/**
* Returns list of scriptlets listed in table for specified platform
*
* @param {"ubo"|"abp"} platform
* @returns {string[]}
*/
const getScriptletsFromTable = (platform) => {
const { scriptlets } = getCompatibilityTable();
return scriptlets.map((item) => item[platform]).filter((item) => !!item);
};
/**
* Returns list of redirects listed in table for specified platform
*
* @param {"ubo"|"abp"} platform
* @returns {string[]}
*/
const getRedirectsFromTable = (platform) => {
const { redirects } = getCompatibilityTable();
return redirects.map((item) => item[platform]).filter((item) => !!item);
};
/**
* @typedef {Object} Diff
* @property {string[]} added added content
* @property {string[]} removed removed content
*/
/**
* Finds a difference between old and new array
*
* @param {Array} oldList
* @param {Array} newList
* @returns {Diff|null}
*/
const getDiff = (oldList, newList) => {
const diff = {
added: [],
removed: [],
};
diff.removed = oldList.filter((item) => !newList.includes(item) && !item.includes(REMOVED_MARKER));
diff.added = newList.filter((item) => !oldList.includes(item));
return (diff.removed.length || diff.added.length) ? diff : null;
};
/**
* Marks removed rules with (removed) and adds new rules to the end of the table
*
* @param {{removed: Array, added: Array}} diff Object with diffs for certain type and platform
* @param {"scriptlets"|"redirects"} ruleType
* @param {"ubo"|"abp"} platform
*/
function markTableWithDiff(diff, ruleType, platform) {
const { removed, added } = diff;
let table = getCompatibilityTable();
const ruleList = table[ruleType].map((item) => {
const rule = item[platform];
if (removed.includes(rule)) {
return {
...item,
[platform]: `${rule} ${REMOVED_MARKER}`,
};
}
return item;
});
added.forEach((item) => ruleList.push({ [platform]: item }));
table = {
...table,
[ruleType]: ruleList,
};
table = JSON.stringify(table, null, 4);
fs.writeFileSync(COMPATIBILITY_TABLE_DATA_PATH, table);
}
/* ************************************************************************
*
* UBO Scriptlets
*
************************************************************************** */
/**
* UBO Scriptlets github raw resources
*/
const UBO_SCRIPTLETS_FILE = 'https://raw.githubusercontent.com/gorhill/uBlock/master/assets/resources/scriptlets.js';
const SCRIPTLETS_START_MARKER = 'Injectable scriptlets';
const DIVIDER_MARKER = 'builtinScriptlets.push';
const COMMENT_MARKER = '//';
const FUNCTION_MARKER = 'function ';
const NAME_MARKER_START = "name: '";
const NAME_MARKER_END = "',";
const ALIASES_MARKER = 'aliases: ';
/**
* Make request to UBO repo(master), parses and returns the list of UBO scriptlets
*/
async function getCurrentUBOScriptlets() {
console.log('Downloading UBO file...');
const { data } = await axios.get(UBO_SCRIPTLETS_FILE);
console.log('UBO done');
const startIndex = data.indexOf(SCRIPTLETS_START_MARKER);
if (startIndex === -1) {
throw new Error('UBO file format has been changed');
}
const names = [];
const chunks = data.slice(startIndex).split(DIVIDER_MARKER);
chunks.forEach((chunk) => {
let name;
let aliases;
const functionDefinitionIndex = chunk.indexOf(FUNCTION_MARKER) || chunk.length;
const scriptletObjectText = chunk.slice(0, functionDefinitionIndex).trim();
const textLines = scriptletObjectText.split(EOL);
for (let i = 0; i < textLines.length; i += 1) {
const line = textLines[i].trim();
if (line.startsWith(COMMENT_MARKER)) {
continue;
}
// parse the name
if (line.startsWith(NAME_MARKER_START)) {
name = line.slice(NAME_MARKER_START.length, line.indexOf(NAME_MARKER_END));
continue;
}
// parse the aliases
if (line.startsWith(ALIASES_MARKER)) {
const aliasesStr = line
.slice(ALIASES_MARKER.length)
// prepare the string for JSON.parse
.replace(/'/g, '"')
.replace(/,?$/, '');
aliases = JSON.parse(aliasesStr);
// 'name' string goes first and 'aliases' string goes after it
// so if aliases are parsed, no need to continue lines iterating
break;
}
}
if (!name) {
return;
}
let namesStr = name;
if (aliases) {
namesStr += ` (${aliases.join(', ')})`;
}
names.push(namesStr);
});
return names;
}
/**
* Check updates for UBO Scriptlets
*/
async function checkForUBOScriptletsUpdates() {
const oldList = getScriptletsFromTable('ubo');
const newList = await getCurrentUBOScriptlets();
const isEqual = areArraysOfStringsEqual(oldList, newList);
const diff = isEqual ? null : getDiff(oldList, newList);
console.log(`UBO Redirects changes ${isEqual ? 'not ' : ''}found`);
return diff;
}
/* ************************************************************************
*
* UBO Redirects
*
************************************************************************** */
/**
* UBO redirects github page
*/
const UBO_REDIRECTS_DIRECTORY_FILE = 'https://raw.githubusercontent.com/gorhill/uBlock/master/src/js/redirect-resources.js';
/**
* Make request to UBO repo(master), parses and returns the list of UBO redirects
*/
async function getCurrentUBORedirects() {
console.log('Downloading UBO page...');
let { data } = await axios.get(UBO_REDIRECTS_DIRECTORY_FILE);
console.log('Done.');
const startTrigger = 'export default new Map([';
const endTrigger = ']);';
const startIndex = data.indexOf(startTrigger);
const endIndex = data.indexOf(endTrigger);
data = data.slice(startIndex, endIndex + endTrigger.length);
const regexp = /\[\s*['|"](\S*)['|"]\s*,\s*{/g;
const names = [];
let result;
// eslint-disable-next-line no-cond-assign
while (result = regexp.exec(data)) {
names.push(result[1]);
}
return names;
}
/**
* Checks updates for UBO redirects
*/
async function checkForUBORedirectsUpdates() {
const oldList = getRedirectsFromTable('ubo');
const newList = await getCurrentUBORedirects();
const isEqual = areArraysOfStringsEqual(oldList, newList);
const diff = isEqual ? null : getDiff(oldList, newList);
console.log(`UBO Redirects changes ${isEqual ? 'not ' : ''}found`);
return diff;
}
/* ************************************************************************
*
* ABP Snippets
*
************************************************************************** */
/**
* ABP Snippets github raw resources
*/
const ABP_SNIPPETS_FILE = 'https://raw.githubusercontent.com/adblockplus/adblockpluscore/master/lib/content/snippets.js';
/**
* Checks for snippets updates
*/
async function getCurrentABPSnippets() { // eslint-disable-line no-unused-vars
console.log('Downloading ABP file...');
const { data } = await axios.get(ABP_SNIPPETS_FILE);
console.log('ABP done.');
const regexp = /exports(\S*)/g;
const names = [];
let result;
// eslint-disable-next-line no-cond-assign
while (result = regexp.exec(data)) {
let [, rawName] = result;
rawName = rawName.replace(/\.|"|'|\[|\]/g, '');
names.push(rawName);
}
return names;
}
/**
* Checks for ABP Snippets updates
*/
async function checkForABPScriptletsUpdates() {
const oldList = getScriptletsFromTable('abp');
// ABP_SNIPPETS_FILE is unavailable
// TODO: fix later, AG-11891
// const newList = await getCurrentABPSnippets();
const newList = oldList;
const isEqual = areArraysOfStringsEqual(oldList, newList);
const diff = isEqual ? null : getDiff(oldList, newList);
console.log(`UBO Redirects changes ${isEqual ? 'not ' : ''}found`);
return diff;
}
/* ************************************************************************
*
* ABP Redirects
*
************************************************************************** */
/**
* ABP redirects github raw resource
*/
const ABP_REDIRECTS_FILE_PATH = 'https://raw.githubusercontent.com/adblockplus/adblockpluscore/master/data/resources.js';
const ABP_REDIRECTS_FILE_SKIP_START = 'exports.resources = ';
/**
* Gets ABP redirects
*/
async function getCurrentABPRedirects() {
console.log('Downloading ABP file...');
const { data: rawData } = await axios.get(ABP_REDIRECTS_FILE_PATH);
const data = JSON.parse(rawData.replace(ABP_REDIRECTS_FILE_SKIP_START, ''));
console.log('ABP done.');
const names = Object.keys(data);
return names;
}
/**
* Checks for ABP redirects updates
*/
async function checkForABPRedirectsUpdates() {
const oldList = getRedirectsFromTable('abp');
const newList = await getCurrentABPRedirects();
const isEqual = areArraysOfStringsEqual(oldList, newList);
const diff = isEqual ? null : getDiff(oldList, newList);
console.log(`UBO Redirects changes ${isEqual ? 'not ' : ''}found`);
return diff;
}
/**
* Entry point
*/
(async function init() {
const UBORedirectsDiff = await checkForUBORedirectsUpdates();
const UBOScriptletsDiff = await checkForUBOScriptletsUpdates();
const ABPRedirectsDiff = await checkForABPRedirectsUpdates();
const ABPScriptletsDiff = await checkForABPScriptletsUpdates();
if (UBORedirectsDiff) {
markTableWithDiff(UBORedirectsDiff, 'redirects', 'ubo');
}
if (UBOScriptletsDiff) {
markTableWithDiff(UBOScriptletsDiff, 'scriptlets', 'ubo');
}
if (ABPRedirectsDiff) {
markTableWithDiff(ABPRedirectsDiff, 'redirects', 'abp');
}
if (ABPScriptletsDiff) {
markTableWithDiff(ABPScriptletsDiff, 'scriptlets', 'abp');
}
const diffs = [UBORedirectsDiff, UBOScriptletsDiff, ABPRedirectsDiff, ABPScriptletsDiff];
if (diffs.some((diff) => !!diff)) {
const removed = diffs
.map((diff) => (diff ? diff.removed.join() : ''))
.filter((item) => !!item)
.join();
const added = diffs
.map((diff) => (diff ? diff.added.join() : ''))
.filter((item) => !!item)
.join();
const message = `
Some sources were changed.
${removed.length ? `Removed: ${removed}.` : ''}
${added.length ? `Added: ${added}.` : ''}
`;
throw new Error(message);
}
}());