From ff32d873375d55f32061d3de4b9cc51dd62b480f Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 21:27:59 +0200 Subject: [PATCH 01/13] Alternating between ascending and descending order Works around Discord taking too long to reload the search page. It sorts messages from start to end while the end to start page is loading so there is no time wasted waiting for pages to load. --- deleteDiscordMessages.user.js | 29 +++++++++++++++++------------ src/ui/undiscord.html | 6 +++--- src/undiscord-core.js | 23 ++++++++++++++--------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index a5f86f08..332e31dc 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -251,7 +251,7 @@
- +
@@ -328,7 +328,7 @@ help
- +
@@ -338,7 +338,7 @@ help
- +

@@ -459,8 +459,9 @@ delCount: 0, failCount: 0, grandTotal: 0, - offset: 0, + offset: {'asc': 0, 'desc': 0}, iterations: 0, + sortOrder: 'asc', _seachResponse: null, _messagesToDelete: [], @@ -487,8 +488,9 @@ delCount: 0, failCount: 0, grandTotal: 0, - offset: 0, + offset: {'asc': 0, 'desc': 0}, iterations: 0, + sortOrder: 'asc', _seachResponse: null, _messagesToDelete: [], @@ -551,6 +553,8 @@ log.verb('Fetching messages...'); // Search messages + this.state.sortOrder = this.state.sortOrder == 'desc' ? 'asc' : 'desc'; + log.verb(`Set sort order to ${this.state.sortOrder} for this search.`); await this.search(); // Process results and find which messages should be deleted @@ -561,7 +565,8 @@ `(Messages in current page: ${this.state._seachResponse.messages.length}`, `To be deleted: ${this.state._messagesToDelete.length}`, `Skipped: ${this.state._skippedMessages.length})`, - `offset: ${this.state.offset}` + `offset (asc): ${this.state.offset['asc']}`, + `offset (desc): ${this.state.offset['desc']}` ); this.printStats(); @@ -582,10 +587,10 @@ else if (this.state._skippedMessages.length > 0) { // There are stuff, but nothing to delete (example a page full of system messages) // check next page until we see a page with nothing in it (end of results). - const oldOffset = this.state.offset; - this.state.offset += this.state._skippedMessages.length; + const oldOffset = this.state.offset[this.state.sortOrder]; + this.state.offset[this.state.sortOrder] += this.state._skippedMessages.length; log.verb('There\'s nothing we can delete on this page, checking next page...'); - log.verb(`Skipped ${this.state._skippedMessages.length} out of ${this.state._seachResponse.messages.length} in this page.`, `(Offset was ${oldOffset}, ajusted to ${this.state.offset})`); + log.verb(`Skipped ${this.state._skippedMessages.length} out of ${this.state._seachResponse.messages.length} in this page.`, `(Offset for ${this.state.sortOrder} was ${oldOffset}, ajusted to ${this.state.offset[this.state.sortOrder]})`); } else { log.verb('Ended because API returned an empty page.'); @@ -657,8 +662,8 @@ ['min_id', this.options.minId ? toSnowflake(this.options.minId) : undefined], ['max_id', this.options.maxId ? toSnowflake(this.options.maxId) : undefined], ['sort_by', 'timestamp'], - ['sort_order', 'desc'], - ['offset', this.state.offset], + ['sort_order', this.state.sortOrder], + ['offset', this.state.offset[this.state.sortOrder]], ['has', this.options.hasLink ? 'link' : undefined], ['has', this.options.hasFile ? 'file' : undefined], ['content', this.options.content || undefined], @@ -825,7 +830,7 @@ // in this case we need to "skip" this message from the next search // otherwise it will come up again in the next page (and fail to delete again) log.warn('Error deleting message (Thread is archived). Will increment offset so we don\'t search this in the next page...'); - this.state.offset++; + this.state.offset[this.state.sortOrder]++; this.state.failCount++; return 'FAIL_SKIP'; // Failed but we will skip it next time } diff --git a/src/ui/undiscord.html b/src/ui/undiscord.html index 455d9e70..613d6864 100644 --- a/src/ui/undiscord.html +++ b/src/ui/undiscord.html @@ -101,7 +101,7 @@

Undiscord

- +
@@ -178,7 +178,7 @@

Undiscord

help
- +
@@ -188,7 +188,7 @@

Undiscord

help
- +

diff --git a/src/undiscord-core.js b/src/undiscord-core.js index 955467f2..77bda127 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -41,8 +41,9 @@ class UndiscordCore { delCount: 0, failCount: 0, grandTotal: 0, - offset: 0, + offset: {'asc': 0, 'desc': 0}, iterations: 0, + sortOrder: 'asc', _seachResponse: null, _messagesToDelete: [], @@ -69,8 +70,9 @@ class UndiscordCore { delCount: 0, failCount: 0, grandTotal: 0, - offset: 0, + offset: {'asc': 0, 'desc': 0}, iterations: 0, + sortOrder: 'asc', _seachResponse: null, _messagesToDelete: [], @@ -133,6 +135,8 @@ class UndiscordCore { log.verb('Fetching messages...'); // Search messages + this.state.sortOrder = this.state.sortOrder == 'desc' ? 'asc' : 'desc'; + log.verb(`Set sort order to ${this.state.sortOrder} for this search.`); await this.search(); // Process results and find which messages should be deleted @@ -143,7 +147,8 @@ class UndiscordCore { `(Messages in current page: ${this.state._seachResponse.messages.length}`, `To be deleted: ${this.state._messagesToDelete.length}`, `Skipped: ${this.state._skippedMessages.length})`, - `offset: ${this.state.offset}` + `offset (asc): ${this.state.offset['asc']}`, + `offset (desc): ${this.state.offset['desc']}` ); this.printStats(); @@ -164,10 +169,10 @@ class UndiscordCore { else if (this.state._skippedMessages.length > 0) { // There are stuff, but nothing to delete (example a page full of system messages) // check next page until we see a page with nothing in it (end of results). - const oldOffset = this.state.offset; - this.state.offset += this.state._skippedMessages.length; + const oldOffset = this.state.offset[this.state.sortOrder]; + this.state.offset[this.state.sortOrder] += this.state._skippedMessages.length; log.verb('There\'s nothing we can delete on this page, checking next page...'); - log.verb(`Skipped ${this.state._skippedMessages.length} out of ${this.state._seachResponse.messages.length} in this page.`, `(Offset was ${oldOffset}, ajusted to ${this.state.offset})`); + log.verb(`Skipped ${this.state._skippedMessages.length} out of ${this.state._seachResponse.messages.length} in this page.`, `(Offset for ${this.state.sortOrder} was ${oldOffset}, ajusted to ${this.state.offset[this.state.sortOrder]})`); } else { log.verb('Ended because API returned an empty page.'); @@ -239,8 +244,8 @@ class UndiscordCore { ['min_id', this.options.minId ? toSnowflake(this.options.minId) : undefined], ['max_id', this.options.maxId ? toSnowflake(this.options.maxId) : undefined], ['sort_by', 'timestamp'], - ['sort_order', 'desc'], - ['offset', this.state.offset], + ['sort_order', this.state.sortOrder], + ['offset', this.state.offset[this.state.sortOrder]], ['has', this.options.hasLink ? 'link' : undefined], ['has', this.options.hasFile ? 'file' : undefined], ['content', this.options.content || undefined], @@ -407,7 +412,7 @@ class UndiscordCore { // in this case we need to "skip" this message from the next search // otherwise it will come up again in the next page (and fail to delete again) log.warn('Error deleting message (Thread is archived). Will increment offset so we don\'t search this in the next page...'); - this.state.offset++; + this.state.offset[this.state.sortOrder]++; this.state.failCount++; return 'FAIL_SKIP'; // Failed but we will skip it next time } From 483cb4fcbfd9095ed31d6a4465e0762e1a6a7d9e Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 21:48:27 +0200 Subject: [PATCH 02/13] Fixed prematurely ending the job on empty page --- deleteDiscordMessages.user.js | 31 +++++++++++++++++++++++++++---- src/undiscord-core.js | 31 +++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 332e31dc..75c40264 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -462,6 +462,9 @@ offset: {'asc': 0, 'desc': 0}, iterations: 0, sortOrder: 'asc', + searchedPages: 0, + totalSkippedMessages: 0, + startEmptyPages: -1, _seachResponse: null, _messagesToDelete: [], @@ -491,6 +494,9 @@ offset: {'asc': 0, 'desc': 0}, iterations: 0, sortOrder: 'asc', + searchedPages: 0, + totalSkippedMessages: 0, + startEmptyPages: -1, _seachResponse: null, _messagesToDelete: [], @@ -556,6 +562,7 @@ this.state.sortOrder = this.state.sortOrder == 'desc' ? 'asc' : 'desc'; log.verb(`Set sort order to ${this.state.sortOrder} for this search.`); await this.search(); + this.state.searchedPages++; // Process results and find which messages should be deleted await this.filterResponse(); @@ -569,6 +576,7 @@ `offset (desc): ${this.state.offset['desc']}` ); this.printStats(); + this.state.totalSkippedMessages += this.state._skippedMessages.length; // Calculate estimated time this.calcEtr(); @@ -576,6 +584,7 @@ // if there are messages to delete, delete them if (this.state._messagesToDelete.length > 0) { + this.state.startEmptyPages = -1; if (await this.confirm() === false) { this.state.running = false; // break out of a job @@ -587,16 +596,30 @@ else if (this.state._skippedMessages.length > 0) { // There are stuff, but nothing to delete (example a page full of system messages) // check next page until we see a page with nothing in it (end of results). + this.state.startEmptyPages = -1; + const oldOffset = this.state.offset[this.state.sortOrder]; this.state.offset[this.state.sortOrder] += this.state._skippedMessages.length; log.verb('There\'s nothing we can delete on this page, checking next page...'); log.verb(`Skipped ${this.state._skippedMessages.length} out of ${this.state._seachResponse.messages.length} in this page.`, `(Offset for ${this.state.sortOrder} was ${oldOffset}, ajusted to ${this.state.offset[this.state.sortOrder]})`); } else { - log.verb('Ended because API returned an empty page.'); - log.verb('[End state]', this.state); - if (isJob) break; // break without stopping if this is part of a job - this.state.running = false; + if (this.state.startEmptyPages == -1) this.state.startEmptyPages = Date.now(); + // if the first page we are searching is empty + // or we've been getting empty page responses for the past 30 seconds (enough for Discord to re-index the pages) + // or (deleted messages + failed to delete + total skipped) >= total messages + // ONLY THEN proceed with ending the job + if (this.state.searchedPages == 1 || (Date.now() - this.state.startEmptyPages) > 30 * 1000 || (this.state.delCount + this.state.failCount + this.state.totalSkippedMessages) >= this.state.grandTotal) { + log.verb('Ended because API returned an empty page.'); + log.verb('[End state]', this.state); + if (isJob) break; // break without stopping if this is part of a job + this.state.running = false; + } else { + // wait 10 seconds for Discord to re-index the search page before retrying + const waitingTime = 10 * 1000; + log.verb(`API returned an empty page, waiting an extra ${(waitingTime / 1000).toFixed(2)}s before searching again...`); + await wait(waitingTime); + } } // wait before next page (fix search page not updating fast enough) diff --git a/src/undiscord-core.js b/src/undiscord-core.js index 77bda127..c267dbba 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -44,6 +44,9 @@ class UndiscordCore { offset: {'asc': 0, 'desc': 0}, iterations: 0, sortOrder: 'asc', + searchedPages: 0, + totalSkippedMessages: 0, + startEmptyPages: -1, _seachResponse: null, _messagesToDelete: [], @@ -73,6 +76,9 @@ class UndiscordCore { offset: {'asc': 0, 'desc': 0}, iterations: 0, sortOrder: 'asc', + searchedPages: 0, + totalSkippedMessages: 0, + startEmptyPages: -1, _seachResponse: null, _messagesToDelete: [], @@ -138,6 +144,7 @@ class UndiscordCore { this.state.sortOrder = this.state.sortOrder == 'desc' ? 'asc' : 'desc'; log.verb(`Set sort order to ${this.state.sortOrder} for this search.`); await this.search(); + this.state.searchedPages++; // Process results and find which messages should be deleted await this.filterResponse(); @@ -151,6 +158,7 @@ class UndiscordCore { `offset (desc): ${this.state.offset['desc']}` ); this.printStats(); + this.state.totalSkippedMessages += this.state._skippedMessages.length; // Calculate estimated time this.calcEtr(); @@ -158,6 +166,7 @@ class UndiscordCore { // if there are messages to delete, delete them if (this.state._messagesToDelete.length > 0) { + this.state.startEmptyPages = -1; if (await this.confirm() === false) { this.state.running = false; // break out of a job @@ -169,16 +178,30 @@ class UndiscordCore { else if (this.state._skippedMessages.length > 0) { // There are stuff, but nothing to delete (example a page full of system messages) // check next page until we see a page with nothing in it (end of results). + this.state.startEmptyPages = -1; + const oldOffset = this.state.offset[this.state.sortOrder]; this.state.offset[this.state.sortOrder] += this.state._skippedMessages.length; log.verb('There\'s nothing we can delete on this page, checking next page...'); log.verb(`Skipped ${this.state._skippedMessages.length} out of ${this.state._seachResponse.messages.length} in this page.`, `(Offset for ${this.state.sortOrder} was ${oldOffset}, ajusted to ${this.state.offset[this.state.sortOrder]})`); } else { - log.verb('Ended because API returned an empty page.'); - log.verb('[End state]', this.state); - if (isJob) break; // break without stopping if this is part of a job - this.state.running = false; + if (this.state.startEmptyPages == -1) this.state.startEmptyPages = Date.now(); + // if the first page we are searching is empty + // or we've been getting empty page responses for the past 30 seconds (enough for Discord to re-index the pages) + // or (deleted messages + failed to delete + total skipped) >= total messages + // ONLY THEN proceed with ending the job + if (this.state.searchedPages == 1 || (Date.now() - this.state.startEmptyPages) > 30 * 1000 || (this.state.delCount + this.state.failCount + this.state.totalSkippedMessages) >= this.state.grandTotal) { + log.verb('Ended because API returned an empty page.'); + log.verb('[End state]', this.state); + if (isJob) break; // break without stopping if this is part of a job + this.state.running = false; + } else { + // wait 10 seconds for Discord to re-index the search page before retrying + const waitingTime = 10 * 1000; + log.verb(`API returned an empty page, waiting an extra ${(waitingTime / 1000).toFixed(2)}s before searching again...`); + await wait(waitingTime); + } } // wait before next page (fix search page not updating fast enough) From 1757e59f7e8f0ebb097d8fa5cce1e2af978f2ad2 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 21:58:30 +0200 Subject: [PATCH 03/13] NaN checks for retry_after, disable delay increase --- deleteDiscordMessages.user.js | 18 ++++++++---------- src/undiscord-core.js | 18 ++++++++---------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 75c40264..65e9a9d3 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -705,8 +705,8 @@ // not indexed yet if (resp.status === 202) { - let w = (await resp.json()).retry_after * 1000; - w = w || this.stats.searchDelay; // Fix retry_after 0 + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 this.stats.throttledCount++; this.stats.throttledTotalTime += w; log.warn(`This channel isn't indexed yet. Waiting ${w}ms for discord to index it...`); @@ -717,14 +717,12 @@ if (!resp.ok) { // searching messages too fast if (resp.status === 429) { - let w = (await resp.json()).retry_after * 1000; - w = w || this.stats.searchDelay; // Fix retry_after 0 + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 this.stats.throttledCount++; this.stats.throttledTotalTime += w; - this.stats.searchDelay += w; // increase delay - w = this.stats.searchDelay; - log.warn(`Being rate limited by the API for ${w}ms! Increasing search delay...`); + log.warn(`Being rate limited by the API for ${w}ms!`); this.printStats(); log.verb(`Cooling down for ${w * 2}ms before retrying...`); @@ -833,11 +831,11 @@ if (!resp.ok) { if (resp.status === 429) { // deleting messages too fast - const w = (await resp.json()).retry_after * 1000; + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.deleteDelay; this.stats.throttledCount++; this.stats.throttledTotalTime += w; - this.options.deleteDelay = w; // increase delay - log.warn(`Being rate limited by the API for ${w}ms! Adjusted delete delay to ${this.options.deleteDelay}ms.`); + log.warn(`Being rate limited by the API for ${w}ms!`); this.printStats(); log.verb(`Cooling down for ${w * 2}ms before retrying...`); await wait(w * 2); diff --git a/src/undiscord-core.js b/src/undiscord-core.js index c267dbba..af31ef6a 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -287,8 +287,8 @@ class UndiscordCore { // not indexed yet if (resp.status === 202) { - let w = (await resp.json()).retry_after * 1000; - w = w || this.stats.searchDelay; // Fix retry_after 0 + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 this.stats.throttledCount++; this.stats.throttledTotalTime += w; log.warn(`This channel isn't indexed yet. Waiting ${w}ms for discord to index it...`); @@ -299,14 +299,12 @@ class UndiscordCore { if (!resp.ok) { // searching messages too fast if (resp.status === 429) { - let w = (await resp.json()).retry_after * 1000; - w = w || this.stats.searchDelay; // Fix retry_after 0 + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 this.stats.throttledCount++; this.stats.throttledTotalTime += w; - this.stats.searchDelay += w; // increase delay - w = this.stats.searchDelay; - log.warn(`Being rate limited by the API for ${w}ms! Increasing search delay...`); + log.warn(`Being rate limited by the API for ${w}ms!`); this.printStats(); log.verb(`Cooling down for ${w * 2}ms before retrying...`); @@ -415,11 +413,11 @@ class UndiscordCore { if (!resp.ok) { if (resp.status === 429) { // deleting messages too fast - const w = (await resp.json()).retry_after * 1000; + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; this.stats.throttledCount++; this.stats.throttledTotalTime += w; - this.options.deleteDelay = w; // increase delay - log.warn(`Being rate limited by the API for ${w}ms! Adjusted delete delay to ${this.options.deleteDelay}ms.`); + log.warn(`Being rate limited by the API for ${w}ms!`); this.printStats(); log.verb(`Cooling down for ${w * 2}ms before retrying...`); await wait(w * 2); From 6e7235b1e96edd0579fdf1e37b901a911e7a15b1 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 22:03:36 +0200 Subject: [PATCH 04/13] Added rate limit prevention --- deleteDiscordMessages.user.js | 19 ++++++++++++++++--- src/undiscord-core.js | 19 ++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 65e9a9d3..bc0bab92 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -678,7 +678,7 @@ let resp; try { - this.beforeRequest(); + await this.beforeRequest(); resp = await fetch(API_SEARCH_URL + 'search?' + queryString([ ['author_id', this.options.authorId || undefined], ['channel_id', (this.options.guildId !== '@me' ? this.options.channelId : undefined) || undefined], @@ -812,7 +812,7 @@ const API_DELETE_URL = `https://discord.com/api/v9/channels/${message.channel_id}/messages/${message.id}`; let resp; try { - this.beforeRequest(); + await this.beforeRequest(); resp = await fetch(API_DELETE_URL, { method: 'DELETE', headers: { @@ -871,7 +871,20 @@ } #beforeTs = 0; // used to calculate latency - beforeRequest() { + #requestLog = []; // used to add any extra delay + async beforeRequest() { + this.#requestLog.push(Date.now()); + this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); + let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right + for(let [maxRequests, timePeriod] of rateLimits){ + if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { + let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); + delay = delay * 1.15 + 300; // adding a buffer and additional wait time + log.verb(`Delaying for an extra ${(delay / 1000).toFixed(2)}s to avoid rate limits...`); + await new Promise(resolve => setTimeout(resolve, delay)); + break; + } + } this.#beforeTs = Date.now(); } afterRequest() { diff --git a/src/undiscord-core.js b/src/undiscord-core.js index af31ef6a..f63a45cb 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -260,7 +260,7 @@ class UndiscordCore { let resp; try { - this.beforeRequest(); + await this.beforeRequest(); resp = await fetch(API_SEARCH_URL + 'search?' + queryString([ ['author_id', this.options.authorId || undefined], ['channel_id', (this.options.guildId !== '@me' ? this.options.channelId : undefined) || undefined], @@ -394,7 +394,7 @@ class UndiscordCore { const API_DELETE_URL = `https://discord.com/api/v9/channels/${message.channel_id}/messages/${message.id}`; let resp; try { - this.beforeRequest(); + await this.beforeRequest(); resp = await fetch(API_DELETE_URL, { method: 'DELETE', headers: { @@ -453,7 +453,20 @@ class UndiscordCore { } #beforeTs = 0; // used to calculate latency - beforeRequest() { + #requestLog = []; // used to add any extra delay + async beforeRequest() { + this.#requestLog.push(Date.now()); + this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); + let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right + for(let [maxRequests, timePeriod] of rateLimits){ + if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { + let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); + delay = delay * 1.15 + 300; // adding a buffer and additional wait time + log.verb(`Delaying for an extra ${(delay / 1000).toFixed(2)}s to avoid rate limits...`); + await new Promise(resolve => setTimeout(resolve, delay)); + break; + } + } this.#beforeTs = Date.now(); } afterRequest() { From 6f76281055342c21ee1147aedcf70ba3cf06bb04 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 22:41:00 +0200 Subject: [PATCH 05/13] Don't crash on search error, includeServers button --- deleteDiscordMessages.user.js | 21 ++++++++++++++++----- src/ui/undiscord.html | 3 +++ src/undiscord-core.js | 14 ++++++++++---- src/undiscord-ui.js | 3 +++ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index bc0bab92..571c52bc 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -228,6 +228,9 @@ After requesting your data from discord, you can import it here.
Select the "messages/index.json" file from the discord archive. +
+ +
@@ -446,6 +449,7 @@ hasLink: null, // Filter messages that contains link hasFile: null, // Filter messages that contains file includeNsfw: null, // Search in NSFW channels + includeServers: null, // Search in server channels includePinned: null, // Delete messages that are pinned pattern: null, // Only delete messages that match the regex (insensitive) searchDelay: null, // Delay each time we fetch for more messages @@ -520,9 +524,12 @@ ...this.options, // keep current options ...job, // override with options for that job }; - - await this.run(true); - if (!this.state.running) break; + if (this.options.guildId !== '@me' && !this.options.includeServers) { + log.verb(`Skipping the channel ${this.options.channelId} as it's a server channel.`); + } else { + await this.run(true); + if (!this.state.running) break; + } log.info('Job ended.', `(${i + 1}/${queue.length})`); this.resetState(); @@ -730,9 +737,10 @@ return await this.search(); } else { - this.state.running = false; log.error(`Error searching messages, API responded with status ${resp.status}!\n`, await resp.json()); - throw resp; + const data = {messages: []}; + this.state._seachResponse = data; + return data; } } const data = await resp.json(); @@ -1488,6 +1496,8 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { const guildId = $('input#guildId').value.trim(); const channelIds = $('input#channelId').value.trim().split(/\s*,\s*/); const includeNsfw = $('input#includeNsfw').checked; + // wipe archive + const includeServers = $('input#includeServers').checked; // filter const content = $('input#search').value.trim(); const hasLink = $('input#hasLink').checked; @@ -1527,6 +1537,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { hasLink, hasFile, includeNsfw, + includeServers, includePinned, pattern, searchDelay, diff --git a/src/ui/undiscord.html b/src/ui/undiscord.html index 613d6864..82fa11fb 100644 --- a/src/ui/undiscord.html +++ b/src/ui/undiscord.html @@ -78,6 +78,9 @@

Undiscord

After requesting your data from discord, you can import it here.
Select the "messages/index.json" file from the discord archive. +
+ +
diff --git a/src/undiscord-core.js b/src/undiscord-core.js index f63a45cb..2c946143 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -28,6 +28,7 @@ class UndiscordCore { hasLink: null, // Filter messages that contains link hasFile: null, // Filter messages that contains file includeNsfw: null, // Search in NSFW channels + includeServers: null, // Search in server channels includePinned: null, // Delete messages that are pinned pattern: null, // Only delete messages that match the regex (insensitive) searchDelay: null, // Delay each time we fetch for more messages @@ -103,8 +104,12 @@ class UndiscordCore { ...job, // override with options for that job }; - await this.run(true); - if (!this.state.running) break; + if (this.options.guildId !== '@me' && !this.options.includeServers) { + log.verb(`Skipping the channel ${this.options.channelId} as it's a server channel.`); + } else { + await this.run(true); + if (!this.state.running) break; + } log.info('Job ended.', `(${i + 1}/${queue.length})`); this.resetState(); @@ -312,9 +317,10 @@ class UndiscordCore { return await this.search(); } else { - this.state.running = false; log.error(`Error searching messages, API responded with status ${resp.status}!\n`, await resp.json()); - throw resp; + const data = {messages: []}; + this.state._seachResponse = data; + return data; } } const data = await resp.json(); diff --git a/src/undiscord-ui.js b/src/undiscord-ui.js index ec0a59b4..597acac4 100644 --- a/src/undiscord-ui.js +++ b/src/undiscord-ui.js @@ -253,6 +253,8 @@ async function startAction() { const guildId = $('input#guildId').value.trim(); const channelIds = $('input#channelId').value.trim().split(/\s*,\s*/); const includeNsfw = $('input#includeNsfw').checked; + // wipe archive + const includeServers = $('input#includeServers').checked; // filter const content = $('input#search').value.trim(); const hasLink = $('input#hasLink').checked; @@ -292,6 +294,7 @@ async function startAction() { hasLink, hasFile, includeNsfw, + includeServers, includePinned, pattern, searchDelay, From c2324c12d4d40545df5868dd3868b705d46c0624 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 22:51:33 +0200 Subject: [PATCH 06/13] Workaround for timer throttling Browsers throttle the JS timers when on another tab, therefore leading to delays being longer than they are supposed to --- deleteDiscordMessages.user.js | 30 +++++++++++++++++++++++++++++- src/utils/helpers.js | 30 +++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 571c52bc..46400a2e 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -419,8 +419,36 @@ var logFn; // custom console.log function const setLogFn = (fn) => logFn = fn; + // Web Worker code as a string + const workerScript = ` + self.addEventListener('message', function(e) { + const ms = e.data; + setTimeout(() => { + self.postMessage('done'); + }, ms); + }); + `; + // Create a Blob URL for the Web Worker + const blob = new Blob([workerScript], { type: 'application/javascript' }); + const workerUrl = URL.createObjectURL(blob); + // Helpers - const wait = async ms => new Promise(done => setTimeout(done, ms)); + const wait = ms => { + return new Promise((resolve, reject) => { + const worker = new Worker(workerUrl); + let start = Date.now(); + worker.postMessage(ms); + worker.addEventListener('message', function(e) { + if (e.data === 'done') { + let delay = Date.now() - start - ms; + if(delay > 100) log.warn(`This action was delayed ${delay}ms more than it should've, make sure you don't have too many tabs open!`); + resolve(); + worker.terminate(); + } + }); + worker.addEventListener('error', reject); + }); + }; const msToHMS = s => `${s / 3.6e6 | 0}h ${(s % 3.6e6) / 6e4 | 0}m ${(s % 6e4) / 1000 | 0}s`; const escapeHTML = html => String(html).replace(/[&<"']/g, m => ({ '&': '&', '<': '<', '"': '"', '\'': ''' })[m]); const redact = str => `${escapeHTML(str)}`; diff --git a/src/utils/helpers.js b/src/utils/helpers.js index 7d029574..ed96f573 100644 --- a/src/utils/helpers.js +++ b/src/utils/helpers.js @@ -1,5 +1,33 @@ +// Web Worker code as a string +const workerScript = ` + self.addEventListener('message', function(e) { + const ms = e.data; + setTimeout(() => { + self.postMessage('done'); + }, ms); + }); +`; +// Create a Blob URL for the Web Worker +const blob = new Blob([workerScript], { type: 'application/javascript' }); +const workerUrl = URL.createObjectURL(blob); + // Helpers -export const wait = async ms => new Promise(done => setTimeout(done, ms)); +const wait = ms => { + return new Promise((resolve, reject) => { + const worker = new Worker(workerUrl); + let start = Date.now(); + worker.postMessage(ms); + worker.addEventListener('message', function(e) { + if (e.data === 'done') { + let delay = Date.now() - start - ms; + if(delay > 100) log.warn(`This action was delayed ${delay}ms more than it should've, make sure you don't have too many tabs open!`); + resolve(); + worker.terminate(); + } + }); + worker.addEventListener('error', reject); + }); +}; export const msToHMS = s => `${s / 3.6e6 | 0}h ${(s % 3.6e6) / 6e4 | 0}m ${(s % 6e4) / 1000 | 0}s`; export const escapeHTML = html => String(html).replace(/[&<"']/g, m => ({ '&': '&', '<': '<', '"': '"', '\'': ''' })[m]); export const redact = str => `${escapeHTML(str)}`; From 53ac50db7f0a1b1c333e5b4255fb8831d4a11982 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 23:14:17 +0200 Subject: [PATCH 07/13] Added option for trimming the log For larger operations with a lot of messages, the tab starts lagging after a couple thousand are deleted --- deleteDiscordMessages.user.js | 14 ++++++++++++++ src/ui/undiscord.html | 1 + src/undiscord-ui.js | 13 +++++++++++++ 3 files changed, 28 insertions(+) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 46400a2e..14a9d4d1 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -397,6 +397,7 @@
@@ -1303,6 +1304,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { undiscordBtn: null, logArea: null, autoScroll: null, + trimLog: null, // progress handler progressMain: null, @@ -1363,6 +1365,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { // cached elements ui.logArea = $('#logArea'); ui.autoScroll = $('#autoScroll'); + ui.trimLog = $('#trimLog'); ui.progressMain = $('#progressBar'); ui.progressIcon = ui.undiscordBtn.querySelector('progress'); ui.percent = $('#progressPercent'); @@ -1456,6 +1459,17 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { function printLog(type = '', args) { ui.logArea.insertAdjacentHTML('beforeend', `
${Array.from(args).map(o => typeof o === 'object' ? JSON.stringify(o, o instanceof Error && Object.getOwnPropertyNames(o)) : o).join('\t')}
`); + + if (ui.trimLog.checked) { + const maxLogEntries = 500; + const logEntries = ui.logArea.querySelectorAll('.log'); + if (logEntries.length > maxLogEntries) { + for (let i = 0; i < (logEntries.length - maxLogEntries); i++) { + logEntries[i].remove(); + } + } + } + if (ui.autoScroll.checked) ui.logArea.querySelector('div:last-child').scrollIntoView(false); if (type==='error') console.error(PREFIX, ...Array.from(args)); } diff --git a/src/ui/undiscord.html b/src/ui/undiscord.html index 82fa11fb..6e666545 100644 --- a/src/ui/undiscord.html +++ b/src/ui/undiscord.html @@ -248,6 +248,7 @@

Undiscord

diff --git a/src/undiscord-ui.js b/src/undiscord-ui.js index 597acac4..2b9636fe 100644 --- a/src/undiscord-ui.js +++ b/src/undiscord-ui.js @@ -32,6 +32,7 @@ const ui = { undiscordBtn: null, logArea: null, autoScroll: null, + trimLog: null, // progress handler progressMain: null, @@ -92,6 +93,7 @@ function initUI() { // cached elements ui.logArea = $('#logArea'); ui.autoScroll = $('#autoScroll'); + ui.trimLog = $('#trimLog'); ui.progressMain = $('#progressBar'); ui.progressIcon = ui.undiscordBtn.querySelector('progress'); ui.percent = $('#progressPercent'); @@ -185,6 +187,17 @@ function initUI() { function printLog(type = '', args) { ui.logArea.insertAdjacentHTML('beforeend', `
${Array.from(args).map(o => typeof o === 'object' ? JSON.stringify(o, o instanceof Error && Object.getOwnPropertyNames(o)) : o).join('\t')}
`); + + if (ui.trimLog.checked) { + const maxLogEntries = 500; + const logEntries = ui.logArea.querySelectorAll('.log'); + if (logEntries.length > maxLogEntries) { + for (let i = 0; i < (logEntries.length - maxLogEntries); i++) { + logEntries[i].remove(); + } + } + } + if (ui.autoScroll.checked) ui.logArea.querySelector('div:last-child').scrollIntoView(false); if (type==='error') console.error(PREFIX, ...Array.from(args)); } From a0ef37404dafa417308002631a137f0d7970c239 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 23:47:27 +0200 Subject: [PATCH 08/13] Minor UI fix --- deleteDiscordMessages.user.js | 2 ++ src/ui/undiscord.html | 2 ++ 2 files changed, 4 insertions(+) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 14a9d4d1..d4e72bb3 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -398,6 +398,8 @@ +
diff --git a/src/ui/undiscord.html b/src/ui/undiscord.html index 6e666545..2b7aa3e6 100644 --- a/src/ui/undiscord.html +++ b/src/ui/undiscord.html @@ -248,6 +248,8 @@

Undiscord

+
From 920e0786e781c170cd1b2ae196bbf9862bf3ad87 Mon Sep 17 00:00:00 2001 From: Voled Date: Sun, 21 Jan 2024 23:53:12 +0200 Subject: [PATCH 09/13] Fixes to pass code factor --- deleteDiscordMessages.user.js | 2 +- src/utils/helpers.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index d4e72bb3..bebbb91c 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -444,7 +444,7 @@ worker.addEventListener('message', function(e) { if (e.data === 'done') { let delay = Date.now() - start - ms; - if(delay > 100) log.warn(`This action was delayed ${delay}ms more than it should've, make sure you don't have too many tabs open!`); + if(delay > 100) console.warn(`This action was delayed ${delay}ms more than it should've, make sure you don't have too many tabs open!`); resolve(); worker.terminate(); } diff --git a/src/utils/helpers.js b/src/utils/helpers.js index ed96f573..16ce9f03 100644 --- a/src/utils/helpers.js +++ b/src/utils/helpers.js @@ -12,7 +12,7 @@ const blob = new Blob([workerScript], { type: 'application/javascript' }); const workerUrl = URL.createObjectURL(blob); // Helpers -const wait = ms => { +export const wait = ms => { return new Promise((resolve, reject) => { const worker = new Worker(workerUrl); let start = Date.now(); @@ -20,7 +20,7 @@ const wait = ms => { worker.addEventListener('message', function(e) { if (e.data === 'done') { let delay = Date.now() - start - ms; - if(delay > 100) log.warn(`This action was delayed ${delay}ms more than it should've, make sure you don't have too many tabs open!`); + if(delay > 100) console.warn(`This action was delayed ${delay}ms more than it should've, make sure you don't have too many tabs open!`); resolve(); worker.terminate(); } From f765db50da23ff28e53fdbadd2fa06b3bd7bd023 Mon Sep 17 00:00:00 2001 From: Voled Date: Thu, 25 Jan 2024 03:34:36 +0200 Subject: [PATCH 10/13] Fixed estimated time remaining calculation --- deleteDiscordMessages.user.js | 2 +- src/undiscord-core.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index bebbb91c..d9c511c4 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -681,7 +681,7 @@ /** Calculate the estimated time remaining based on the current stats */ calcEtr() { - this.stats.etr = (this.options.searchDelay * Math.round(this.state.grandTotal / 25)) + ((this.options.deleteDelay + this.stats.avgPing) * this.state.grandTotal); + this.stats.etr = (this.options.searchDelay + this.stats.avgPing) * Math.round((this.state.grandTotal - this.state.delCount) / 25) + (this.options.deleteDelay + this.stats.avgPing) * (this.state.grandTotal - this.state.delCount); } /** As for confirmation in the beggining process */ diff --git a/src/undiscord-core.js b/src/undiscord-core.js index 2c946143..172a4ada 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -230,7 +230,7 @@ class UndiscordCore { /** Calculate the estimated time remaining based on the current stats */ calcEtr() { - this.stats.etr = (this.options.searchDelay * Math.round(this.state.grandTotal / 25)) + ((this.options.deleteDelay + this.stats.avgPing) * this.state.grandTotal); + this.stats.etr = (this.options.searchDelay + this.stats.avgPing) * Math.round((this.state.grandTotal - this.state.delCount) / 25) + (this.options.deleteDelay + this.stats.avgPing) * (this.state.grandTotal - this.state.delCount); } /** As for confirmation in the beggining process */ From db69f9f85f87b2e5245c9eb191ac521650e61c93 Mon Sep 17 00:00:00 2001 From: Voled Date: Sat, 27 Jan 2024 16:33:11 +0200 Subject: [PATCH 11/13] Added support for servers in wipe archive --- deleteDiscordMessages.user.js | 85 ++++++++++++++++++++++++++++++----- src/undiscord-core.js | 78 ++++++++++++++++++++++++++++---- src/undiscord-ui.js | 6 +-- 3 files changed, 145 insertions(+), 24 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index d9c511c4..486cb31d 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -555,12 +555,9 @@ ...this.options, // keep current options ...job, // override with options for that job }; - if (this.options.guildId !== '@me' && !this.options.includeServers) { - log.verb(`Skipping the channel ${this.options.channelId} as it's a server channel.`); - } else { - await this.run(true); - if (!this.state.running) break; - } + + await this.run(true); + if (!this.state.running) break; log.info('Job ended.', `(${i + 1}/${queue.length})`); this.resetState(); @@ -580,6 +577,18 @@ this.stats.startTime = new Date(); log.success(`\nStarted at ${this.stats.startTime.toLocaleString()}`); + if (this.onStart) this.onStart(this.state, this.stats); + + if (!this.options.guildId) { + log.verb('Fetching channel info...'); + await this.fetchChannelInfo(); + } + if (!this.options.guildId) return; // message is handled in fetchChannelInfo + if (isJob && this.options.guildId !== '@me' && !this.options.includeServers) { + log.warn(`Skipping the channel ${this.options.channelId} as it's a server channel.`); + return; + } + log.debug( `authorId = "${redact(this.options.authorId)}"`, `guildId = "${redact(this.options.guildId)}"`, @@ -590,8 +599,6 @@ `hasFile = ${!!this.options.hasFile}`, ); - if (this.onStart) this.onStart(this.state, this.stats); - do { this.state.iterations++; @@ -709,6 +716,60 @@ } } + async fetchChannelInfo() { + let API_CHANNEL_URL = `https://discord.com/api/v9/channels/${this.options.channelId}`; + + let resp; + try { + await this.beforeRequest(); + resp = await fetch(API_CHANNEL_URL, { + headers: { + 'Authorization': this.options.authToken, + } + }); + this.afterRequest(); + } catch (err) { + this.state.running = false; + log.error('Channel request threw an error:', err); + throw err; + } + + // not indexed yet + if (resp.status === 202) { + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 + this.stats.throttledCount++; + this.stats.throttledTotalTime += w; + log.warn(`This channel isn't indexed yet. Waiting ${w}ms for discord to index it...`); + await wait(w); + return await this.fetchChannelInfo(); + } + + if (!resp.ok) { + // rate limit + if (resp.status === 429) { + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 + + this.stats.throttledCount++; + this.stats.throttledTotalTime += w; + log.warn(`Being rate limited by the API for ${w}ms!`); + this.printStats(); + log.verb(`Cooling down for ${w * 2}ms before retrying...`); + + await wait(w * 2); + return await this.fetchChannelInfo(); + } + else { + log.error(`Error fetching the channel, API responded with status ${resp.status}!\n`, await resp.json()); + return {}; + } + } + const data = await resp.json(); + this.options.guildId = data.guild_id ?? '@me'; + return data; + } + async search() { let API_SEARCH_URL; if (this.options.guildId === '@me') API_SEARCH_URL = `https://discord.com/api/v9/channels/${this.options.channelId}/messages/`; // DMs @@ -915,7 +976,7 @@ this.#requestLog.push(Date.now()); this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right - for(let [maxRequests, timePeriod] of rateLimits){ + for (let [maxRequests, timePeriod] of rateLimits) { if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); delay = delay * 1.15 + 300; // adding a buffer and additional wait time @@ -1435,9 +1496,9 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { // Get channel id field to set it later const channelIdField = $('input#channelId'); - // Force the guild id to be ourself (@me) + // Force the guild id to be 'null' (placeholder value) const guildIdField = $('input#guildId'); - guildIdField.value = '@me'; + guildIdField.value = 'null'; // Set author id in case its not set already $('input#authorId').value = getAuthorId(); @@ -1590,7 +1651,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { }; if (channelIds.length > 1) { const jobs = channelIds.map(ch => ({ - guildId: guildId, + guildId: null, channelId: ch, })); diff --git a/src/undiscord-core.js b/src/undiscord-core.js index 172a4ada..9d22200f 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -104,12 +104,8 @@ class UndiscordCore { ...job, // override with options for that job }; - if (this.options.guildId !== '@me' && !this.options.includeServers) { - log.verb(`Skipping the channel ${this.options.channelId} as it's a server channel.`); - } else { - await this.run(true); - if (!this.state.running) break; - } + await this.run(true); + if (!this.state.running) break; log.info('Job ended.', `(${i + 1}/${queue.length})`); this.resetState(); @@ -129,6 +125,18 @@ class UndiscordCore { this.stats.startTime = new Date(); log.success(`\nStarted at ${this.stats.startTime.toLocaleString()}`); + if (this.onStart) this.onStart(this.state, this.stats); + + if (!this.options.guildId) { + log.verb('Fetching channel info...'); + await this.fetchChannelInfo(); + } + if (!this.options.guildId) return; // message is handled in fetchChannelInfo + if (isJob && this.options.guildId !== '@me' && !this.options.includeServers) { + log.warn(`Skipping the channel ${this.options.channelId} as it's a server channel.`); + return; + } + log.debug( `authorId = "${redact(this.options.authorId)}"`, `guildId = "${redact(this.options.guildId)}"`, @@ -139,8 +147,6 @@ class UndiscordCore { `hasFile = ${!!this.options.hasFile}`, ); - if (this.onStart) this.onStart(this.state, this.stats); - do { this.state.iterations++; @@ -258,6 +264,60 @@ class UndiscordCore { } } + async fetchChannelInfo() { + let API_CHANNEL_URL = `https://discord.com/api/v9/channels/${this.options.channelId}`; + + let resp; + try { + await this.beforeRequest(); + resp = await fetch(API_CHANNEL_URL, { + headers: { + 'Authorization': this.options.authToken, + } + }); + this.afterRequest(); + } catch (err) { + this.state.running = false; + log.error('Channel request threw an error:', err); + throw err; + } + + // not indexed yet + if (resp.status === 202) { + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 + this.stats.throttledCount++; + this.stats.throttledTotalTime += w; + log.warn(`This channel isn't indexed yet. Waiting ${w}ms for discord to index it...`); + await wait(w); + return await this.fetchChannelInfo(); + } + + if (!resp.ok) { + // rate limit + if (resp.status === 429) { + let w = (await resp.json()).retry_after; + w = !isNaN(w) ? w * 1000 : this.stats.searchDelay; // Fix retry_after 0 + + this.stats.throttledCount++; + this.stats.throttledTotalTime += w; + log.warn(`Being rate limited by the API for ${w}ms!`); + this.printStats(); + log.verb(`Cooling down for ${w * 2}ms before retrying...`); + + await wait(w * 2); + return await this.fetchChannelInfo(); + } + else { + log.error(`Error fetching the channel, API responded with status ${resp.status}!\n`, await resp.json()); + return {}; + } + } + const data = await resp.json(); + this.options.guildId = data.guild_id ?? '@me'; + return data; + } + async search() { let API_SEARCH_URL; if (this.options.guildId === '@me') API_SEARCH_URL = `https://discord.com/api/v9/channels/${this.options.channelId}/messages/`; // DMs @@ -464,7 +524,7 @@ class UndiscordCore { this.#requestLog.push(Date.now()); this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right - for(let [maxRequests, timePeriod] of rateLimits){ + for (let [maxRequests, timePeriod] of rateLimits) { if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); delay = delay * 1.15 + 300; // adding a buffer and additional wait time diff --git a/src/undiscord-ui.js b/src/undiscord-ui.js index 2b9636fe..05e577dd 100644 --- a/src/undiscord-ui.js +++ b/src/undiscord-ui.js @@ -161,9 +161,9 @@ function initUI() { // Get channel id field to set it later const channelIdField = $('input#channelId'); - // Force the guild id to be ourself (@me) + // Force the guild id to be 'null' (placeholder value) const guildIdField = $('input#guildId'); - guildIdField.value = '@me'; + guildIdField.value = 'null'; // Set author id in case its not set already $('input#authorId').value = getAuthorId(); @@ -333,7 +333,7 @@ async function startAction() { // multiple channels else if (channelIds.length > 1) { const jobs = channelIds.map(ch => ({ - guildId: guildId, + guildId: null, channelId: ch, })); From 208e39a24259b7e346a642f9133fc438a17a02ae Mon Sep 17 00:00:00 2001 From: Voled Date: Wed, 31 Jan 2024 18:40:49 +0200 Subject: [PATCH 12/13] Added an option for rate limit prevention --- deleteDiscordMessages.user.js | 21 +++++++++++++-------- src/ui/undiscord.html | 1 + src/undiscord-core.js | 18 ++++++++++-------- src/undiscord-ui.js | 2 ++ 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 486cb31d..8c9aca8e 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -350,6 +350,7 @@ Use the help link for more information. +
@@ -975,14 +976,16 @@ async beforeRequest() { this.#requestLog.push(Date.now()); this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); - let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right - for (let [maxRequests, timePeriod] of rateLimits) { - if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { - let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); - delay = delay * 1.15 + 300; // adding a buffer and additional wait time - log.verb(`Delaying for an extra ${(delay / 1000).toFixed(2)}s to avoid rate limits...`); - await new Promise(resolve => setTimeout(resolve, delay)); - break; + if (ui.rateLimitPrevention.checked) { + let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right + for (let [maxRequests, timePeriod] of rateLimits) { + if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { + let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); + delay = delay * 1.15 + 300; // adding a buffer and additional wait time + log.verb(`Delaying for an extra ${(delay / 1000).toFixed(2)}s to avoid rate limits...`); + await new Promise(resolve => setTimeout(resolve, delay)); + break; + } } } this.#beforeTs = Date.now(); @@ -1368,6 +1371,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { logArea: null, autoScroll: null, trimLog: null, + rateLimitPrevention: null, // progress handler progressMain: null, @@ -1432,6 +1436,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { ui.progressMain = $('#progressBar'); ui.progressIcon = ui.undiscordBtn.querySelector('progress'); ui.percent = $('#progressPercent'); + ui.rateLimitPrevention = $('#rateLimitPrevention'); // register event listeners $('#hide').onclick = toggleWindow; diff --git a/src/ui/undiscord.html b/src/ui/undiscord.html index 2b7aa3e6..d5b05e9a 100644 --- a/src/ui/undiscord.html +++ b/src/ui/undiscord.html @@ -200,6 +200,7 @@

Undiscord

Use the help link for more information.
+
diff --git a/src/undiscord-core.js b/src/undiscord-core.js index 9d22200f..19997f90 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -523,14 +523,16 @@ class UndiscordCore { async beforeRequest() { this.#requestLog.push(Date.now()); this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); - let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right - for (let [maxRequests, timePeriod] of rateLimits) { - if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { - let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); - delay = delay * 1.15 + 300; // adding a buffer and additional wait time - log.verb(`Delaying for an extra ${(delay / 1000).toFixed(2)}s to avoid rate limits...`); - await new Promise(resolve => setTimeout(resolve, delay)); - break; + if (ui.rateLimitPrevention.checked) { + let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right + for (let [maxRequests, timePeriod] of rateLimits) { + if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { + let delay = timePeriod * 1000 - (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]); + delay = delay * 1.15 + 300; // adding a buffer and additional wait time + log.verb(`Delaying for an extra ${(delay / 1000).toFixed(2)}s to avoid rate limits...`); + await new Promise(resolve => setTimeout(resolve, delay)); + break; + } } } this.#beforeTs = Date.now(); diff --git a/src/undiscord-ui.js b/src/undiscord-ui.js index 05e577dd..fe3dc374 100644 --- a/src/undiscord-ui.js +++ b/src/undiscord-ui.js @@ -33,6 +33,7 @@ const ui = { logArea: null, autoScroll: null, trimLog: null, + rateLimitPrevention: null, // progress handler progressMain: null, @@ -97,6 +98,7 @@ function initUI() { ui.progressMain = $('#progressBar'); ui.progressIcon = ui.undiscordBtn.querySelector('progress'); ui.percent = $('#progressPercent'); + ui.rateLimitPrevention = $('#rateLimitPrevention'); // register event listeners $('#hide').onclick = toggleWindow; From cd91fc71ef9c54eed66a42f6139e006dbf04616b Mon Sep 17 00:00:00 2001 From: Voled Date: Wed, 31 Jan 2024 19:04:02 +0200 Subject: [PATCH 13/13] Fixed rate limit prevention option --- deleteDiscordMessages.user.js | 12 ++++++++---- src/undiscord-core.js | 3 ++- src/undiscord-ui.js | 9 ++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/deleteDiscordMessages.user.js b/deleteDiscordMessages.user.js index 8c9aca8e..f13f295f 100644 --- a/deleteDiscordMessages.user.js +++ b/deleteDiscordMessages.user.js @@ -486,6 +486,7 @@ pattern: null, // Only delete messages that match the regex (insensitive) searchDelay: null, // Delay each time we fetch for more messages deleteDelay: null, // Delay between each delete operation + rateLimitPrevention: null, // Whether rate limit prevention is enabled or not maxAttempt: 2, // Attempts to delete a single message if it fails askForConfirmation: true, }; @@ -976,7 +977,7 @@ async beforeRequest() { this.#requestLog.push(Date.now()); this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); - if (ui.rateLimitPrevention.checked) { + if (this.options.rateLimitPrevention) { let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right for (let [maxRequests, timePeriod] of rateLimits) { if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { @@ -1371,7 +1372,6 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { logArea: null, autoScroll: null, trimLog: null, - rateLimitPrevention: null, // progress handler progressMain: null, @@ -1436,7 +1436,6 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { ui.progressMain = $('#progressBar'); ui.progressIcon = ui.undiscordBtn.querySelector('progress'); ui.percent = $('#progressPercent'); - ui.rateLimitPrevention = $('#rateLimitPrevention'); // register event listeners $('#hide').onclick = toggleWindow; @@ -1473,7 +1472,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { }; $('button#getToken').onclick = () => $('input#token').value = fillToken(); - // sync delays + // sync advanced settings $('input#searchDelay').onchange = (e) => { const v = parseInt(e.target.value); if (v) undiscordCore.options.searchDelay = v; @@ -1482,6 +1481,9 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { const v = parseInt(e.target.value); if (v) undiscordCore.options.deleteDelay = v; }; + $('input#rateLimitPrevention').onchange = (e) => { + undiscordCore.options.rateLimitPrevention = e.target.checked ?? false; + }; $('input#searchDelay').addEventListener('input', (event) => { $('div#searchDelayValue').textContent = event.target.value + 'ms'; @@ -1623,6 +1625,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { //advanced const searchDelay = parseInt($('input#searchDelay').value.trim()); const deleteDelay = parseInt($('input#deleteDelay').value.trim()); + const rateLimitPrevention = $('input#rateLimitPrevention').checked; // token const authToken = $('input#token').value.trim() || fillToken(); @@ -1652,6 +1655,7 @@ body.undiscord-pick-message.after [id^="message-content-"]:hover::after { pattern, searchDelay, deleteDelay, + rateLimitPrevention, // maxAttempt: 2, }; if (channelIds.length > 1) { diff --git a/src/undiscord-core.js b/src/undiscord-core.js index 19997f90..489c0ee8 100644 --- a/src/undiscord-core.js +++ b/src/undiscord-core.js @@ -33,6 +33,7 @@ class UndiscordCore { pattern: null, // Only delete messages that match the regex (insensitive) searchDelay: null, // Delay each time we fetch for more messages deleteDelay: null, // Delay between each delete operation + rateLimitPrevention: null, // Whether rate limit prevention is enabled or not maxAttempt: 2, // Attempts to delete a single message if it fails askForConfirmation: true, }; @@ -523,7 +524,7 @@ class UndiscordCore { async beforeRequest() { this.#requestLog.push(Date.now()); this.#requestLog = this.#requestLog.filter(timestamp => (Date.now() - timestamp) < 60 * 1000); - if (ui.rateLimitPrevention.checked) { + if (this.options.rateLimitPrevention) { let rateLimits = [[45, 60], [4, 5]]; // todo: confirm, testing shows these are right for (let [maxRequests, timePeriod] of rateLimits) { if (this.#requestLog.length >= maxRequests && (Date.now() - this.#requestLog[this.#requestLog.length - maxRequests]) < timePeriod * 1000) { diff --git a/src/undiscord-ui.js b/src/undiscord-ui.js index fe3dc374..076acc8a 100644 --- a/src/undiscord-ui.js +++ b/src/undiscord-ui.js @@ -33,7 +33,6 @@ const ui = { logArea: null, autoScroll: null, trimLog: null, - rateLimitPrevention: null, // progress handler progressMain: null, @@ -98,7 +97,6 @@ function initUI() { ui.progressMain = $('#progressBar'); ui.progressIcon = ui.undiscordBtn.querySelector('progress'); ui.percent = $('#progressPercent'); - ui.rateLimitPrevention = $('#rateLimitPrevention'); // register event listeners $('#hide').onclick = toggleWindow; @@ -135,7 +133,7 @@ function initUI() { }; $('button#getToken').onclick = () => $('input#token').value = fillToken(); - // sync delays + // sync advanced settings $('input#searchDelay').onchange = (e) => { const v = parseInt(e.target.value); if (v) undiscordCore.options.searchDelay = v; @@ -144,6 +142,9 @@ function initUI() { const v = parseInt(e.target.value); if (v) undiscordCore.options.deleteDelay = v; }; + $('input#rateLimitPrevention').onchange = (e) => { + undiscordCore.options.rateLimitPrevention = e.target.checked ?? false; + }; $('input#searchDelay').addEventListener('input', (event) => { $('div#searchDelayValue').textContent = event.target.value + 'ms'; @@ -285,6 +286,7 @@ async function startAction() { //advanced const searchDelay = parseInt($('input#searchDelay').value.trim()); const deleteDelay = parseInt($('input#deleteDelay').value.trim()); + const rateLimitPrevention = $('input#rateLimitPrevention').checked; // token const authToken = $('input#token').value.trim() || fillToken(); @@ -314,6 +316,7 @@ async function startAction() { pattern, searchDelay, deleteDelay, + rateLimitPrevention, // maxAttempt: 2, };