Skip to content

Commit

Permalink
Update cleansing rules for RSS TL feed and homedir for Mac
Browse files Browse the repository at this point in the history 
(cherry picked from commit e5ff4aafa3f0b855fec332788e9fc490a03dfce3)

Closes #2593
  • Loading branch information
@mynameisbogdan
mynameisbogdan committed Jun 15, 2023
1 parent c7eaf1e commit 4e493b7
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 49 deletions.
8 changes: 7 additions & 1 deletion src/NzbDrone.Common.Test/InstrumentationTests/CleanseLogMessageFixture.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ public class CleanseLogMessageFixture
// Indexer Urls
[TestCase(@"https://iptorrents.com/torrents/rss?u=mySecret;tp=mySecret;l5;download")]
[TestCase(@"http://rss.torrentleech.org/mySecret")]
[TestCase(@"http://rss.torrentleech.org/rss/download/12345/01233210/filename.torrent")]
[TestCase(@"https://rss24h.torrentleech.org/mySecret")]
[TestCase(@"http://rss.torrentleech.org/rss/download/12345/01233210/file.name-RLSGRP.torrent")]
[TestCase(@"https://www.torrentleech.org/rss/download/12345/01233210/file.name-RLSGRP.torrent")]
[TestCase(@"http://www.bitmetv.org/rss.php?uid=mySecret&passkey=mySecret")]
[TestCase(@"https://rss.omgwtfnzbs.org/rss-search.php?catid=19,20&user=Readarr&api=mySecret&eng=1")]
[TestCase(@"https://dognzb.cr/fetch/2b51db35e1912ffc138825a12b9933d2/2b51db35e1910123321025a12b9933d2")]
Expand Down Expand Up @@ -44,6 +46,7 @@ public class CleanseLogMessageFixture
// Deluge
[TestCase(@",{""download_location"": ""C:\Users\\mySecret mySecret\\Downloads""}")]
[TestCase(@",{""download_location"": ""/home/mySecret/Downloads""}")]
[TestCase(@",{""download_location"": ""/Users/mySecret/Downloads""}")]
[TestCase(@"auth.login(""mySecret"")")]

// Download Station
Expand All @@ -56,8 +59,11 @@ public class CleanseLogMessageFixture

// Internal
[TestCase(@"OutputPath=/home/mySecret/Downloads")]
[TestCase(@"OutputPath=/Users/mySecret/Downloads")]
[TestCase("Hardlinking episode file: /home/mySecret/Downloads to /media/abc.mkv")]
[TestCase("Hardlinking episode file: /Users/mySecret/Downloads to /media/abc.mkv")]
[TestCase("Hardlink '/home/mySecret/Downloads/abs.mkv' to '/media/abc.mkv' failed.")]
[TestCase("Hardlink '/Users/mySecret/Downloads/abs.mkv' to '/media/abc.mkv' failed.")]
[TestCase("https://notifiarr.com/notifier.php: api=1234530f-422f-4aac-b6b3-01233210aaaa&radarr_health_issue_message=Download")]
[TestCase("/readarr/signalr/messages/negotiate?access_token=1234530f422f4aacb6b301233210aaaa&negotiateVersion=1")]
[TestCase(@"[Info] MigrationController: *** Migrating Database=readarr-main;Host=postgres14;Username=mySecret;Password=mySecret;Port=5432;Enlist=False ***")]
Expand Down
95 changes: 47 additions & 48 deletions src/NzbDrone.Common/Instrumentation/CleanseLogMessage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,56 +7,56 @@ namespace NzbDrone.Common.Instrumentation
{
public class CleanseLogMessage
{
private static readonly Regex[] CleansingRules = new[]
{
// Url
new Regex(@"(?<=\?|&|: )((?:api|auth|pass)?key|(?:access[-_]?)?token|auth|user|uid|api|[a-z_]*apikey|account|passwd)=(?<secret>[^&=""]+?)(?=[ ""&=]|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"(?<=\?|&)[^=]*?(username|password)=(?<secret>[^&=]+?)(?= |&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"torrentleech\.org/(?!rss)(?<secret>[0-9a-z]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"torrentleech\.org/rss/download/[0-9]+/(?<secret>[0-9a-z]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"iptorrents\.com/[/a-z0-9?&;]*?(?:[?&;](u|tp)=(?<secret>[^&=;]+?))+(?= |;|&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"/fetch/[a-z0-9]{32}/(?<secret>[a-z0-9]{32})", RegexOptions.Compiled),
new Regex(@"getnzb.*?(?<=\?|&)(r)=(?<secret>[^&=]+?)(?= |&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"\b(\w*)?(_?(?<!use|get_)token|username|passwo?rd)=(?<secret>[^&=]+?)(?= |&|$|;)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
private static readonly Regex[] CleansingRules =
{
// Url
new (@"(?<=\?|&|: )((?:api|auth|pass)?key|(?:access[-_]?)?token|auth|user|uid|api|[a-z_]*apikey|account|passwd)=(?<secret>[^&=""]+?)(?=[ ""&=]|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"(?<=\?|&)[^=]*?(username|password)=(?<secret>[^&=]+?)(?= |&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"rss(24h)?\.torrentleech\.org/(?!rss)(?<secret>[0-9a-z]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"torrentleech\.org/rss/download/[0-9]+/(?<secret>[0-9a-z]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"iptorrents\.com/[/a-z0-9?&;]*?(?:[?&;](u|tp)=(?<secret>[^&=;]+?))+(?= |;|&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"/fetch/[a-z0-9]{32}/(?<secret>[a-z0-9]{32})", RegexOptions.Compiled),
new (@"getnzb.*?(?<=\?|&)(r)=(?<secret>[^&=]+?)(?= |&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"\b(\w*)?(_?(?<!use|get_)token|username|passwo?rd)=(?<secret>[^&=]+?)(?= |&|$|;)", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// Trackers Announce Keys; Designed for Qbit Json; should work for all in theory
new Regex(@"announce(\.php)?(/|%2f|%3fpasskey%3d)(?<secret>[a-z0-9]{16,})|(?<secret>[a-z0-9]{16,})(/|%2f)announce"),
// Trackers Announce Keys; Designed for Qbit Json; should work for all in theory
new (@"announce(\.php)?(/|%2f|%3fpasskey%3d)(?<secret>[a-z0-9]{16,})|(?<secret>[a-z0-9]{16,})(/|%2f)announce"),

// Path
new Regex(@"C:\\Users\\(?<secret>[^\""]+?)(\\|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"/home/(?<secret>[^/""]+?)(/|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// Path
new (@"C:\\Users\\(?<secret>[^\""]+?)(\\|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"/(home|Users)/(?<secret>[^/""]+?)(/|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// NzbGet
new Regex(@"""Name""\s*:\s*""[^""]*(username|password)""\s*,\s*""Value""\s*:\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// NzbGet
new (@"""Name""\s*:\s*""[^""]*(username|password)""\s*,\s*""Value""\s*:\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// Sabnzbd
new Regex(@"""[^""]*(username|password|api_?key|nzb_key)""\s*:\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"""email_(account|to|from|pwd)""\s*:\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// Sabnzbd
new (@"""[^""]*(username|password|api_?key|nzb_key)""\s*:\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"""email_(account|to|from|pwd)""\s*:\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// uTorrent
new Regex(@"\[""[a-z._]*(username|password)"",\d,""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"\[""(boss_key|boss_key_salt|proxy\.proxy)"",\d,""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// uTorrent
new (@"\[""[a-z._]*(username|password)"",\d,""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"\[""(boss_key|boss_key_salt|proxy\.proxy)"",\d,""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// Deluge
new Regex(@"auth.login\(""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// Deluge
new (@"auth.login\(""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// BroadcastheNet
new Regex(@"""?method""?\s*:\s*""(getTorrents)"",\s*""?params""?\s*:\s*\[\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"getTorrents\(""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new Regex(@"(?<=\?|&)(authkey|torrent_pass)=(?<secret>[^&=]+?)(?=""|&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// BroadcastheNet
new (@"""?method""?\s*:\s*""(getTorrents)"",\s*""?params""?\s*:\s*\[\s*""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"getTorrents\(""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
new (@"(?<=\?|&)(authkey|torrent_pass)=(?<secret>[^&=]+?)(?=""|&|$)", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// Good Reads
new Regex(@"(?<=""(token|tokensecret)"":\s)""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// Good Reads
new (@"(?<=""(token|tokensecret)"":\s)""(?<secret>[^""]+?)""", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// Webhooks
// Notifiarr
new Regex(@"api/v[0-9]/notification/readarr/(?<secret>[\w-]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase),
// Webhooks
// Notifiarr
new (@"api/v[0-9]/notification/readarr/(?<secret>[\w-]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase),

// Discord
new Regex(@"discord.com/api/webhooks/((?<secret>[\w-]+)/)?(?<secret>[\w-]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase)
};
// Discord
new (@"discord.com/api/webhooks/((?<secret>[\w-]+)/)?(?<secret>[\w-]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase)
};

private static readonly Regex CleanseRemoteIPRegex = new Regex(@"(?:Auth-\w+(?<!Failure|Unauthorized) ip|from) (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", RegexOptions.Compiled);
private static readonly Regex CleanseRemoteIPRegex = new (@"(?:Auth-\w+(?<!Failure|Unauthorized) ip|from) (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})", RegexOptions.Compiled);

public static string Cleanse(string message)
{
Expand All @@ -68,15 +68,15 @@ public static string Cleanse(string message)
foreach (var regex in CleansingRules)
{
message = regex.Replace(message, m =>
{
var value = m.Value;
foreach (var capture in m.Groups["secret"].Captures.OfType<Capture>().Reverse())
{
var value = m.Value;
foreach (var capture in m.Groups["secret"].Captures.OfType<Capture>().Reverse())
{
value = value.Replace(capture.Index - m.Index, capture.Length, "(removed)");
}
return value;
});
value = value.Replace(capture.Index - m.Index, capture.Length, "(removed)");
}
return value;
});
}

message = CleanseRemoteIPRegex.Replace(message, CleanseRemoteIP);
Expand All @@ -87,7 +87,6 @@ public static string Cleanse(string message)
private static string CleanseRemoteIP(Match match)
{
var group = match.Groups[1];
var valueAll = match.Value;
var valueIP = group.Value;

if (IPAddress.TryParse(valueIP, out var address) && !address.IsLocalAddress())
Expand Down

0 comments on commit 4e493b7

Please sign in to comment.