From 584499b55e418774f0b3d5b9efcae1085f4461d7 Mon Sep 17 00:00:00 2001 From: n0099 Date: Sat, 11 May 2024 06:28:34 +0800 Subject: [PATCH] * rename abstract class `BaseCrawlFacade`, `BasePostSaver` & `BasePostParser` to remove `Base-` prefix @ c#/crawler --- c#/crawler/src/EntryPoint.cs | 4 ++-- .../src/Tieba/Crawl/Crawler/ThreadLateCrawler.cs | 2 +- .../{BaseCrawlFacade.cs => CrawlFacade.cs} | 16 ++++++++-------- .../src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs | 2 +- .../Tieba/Crawl/Facade/SubReplyCrawlFacade.cs | 2 +- .../src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs | 2 +- .../Post/{BasePostParser.cs => PostParser.cs} | 2 +- .../src/Tieba/Crawl/Parser/Post/ReplyParser.cs | 2 +- .../Tieba/Crawl/Parser/Post/SubReplyParser.cs | 2 +- .../src/Tieba/Crawl/Parser/Post/ThreadParser.cs | 2 +- .../Post/{BasePostSaver.cs => PostSaver.cs} | 4 ++-- .../src/Tieba/Crawl/Saver/Post/ReplySaver.cs | 2 +- .../src/Tieba/Crawl/Saver/Post/SubReplySaver.cs | 2 +- .../src/Tieba/Crawl/Saver/Post/ThreadSaver.cs | 2 +- 14 files changed, 23 insertions(+), 23 deletions(-) rename c#/crawler/src/Tieba/Crawl/Facade/{BaseCrawlFacade.cs => CrawlFacade.cs} (93%) rename c#/crawler/src/Tieba/Crawl/Parser/Post/{BasePostParser.cs => PostParser.cs} (96%) rename c#/crawler/src/Tieba/Crawl/Saver/Post/{BasePostSaver.cs => PostSaver.cs} (94%) diff --git a/c#/crawler/src/EntryPoint.cs b/c#/crawler/src/EntryPoint.cs index a618d16e..a4bce0dd 100644 --- a/c#/crawler/src/EntryPoint.cs +++ b/c#/crawler/src/EntryPoint.cs @@ -36,8 +36,8 @@ protected override void ConfigureContainer(HostBuilderContext context, Container { builder.RegisterImplementsOfBaseTypes(typeof(EntryPoint).Assembly, [ - typeof(BaseCrawler<,>), typeof(BaseCrawlFacade<,,,>), - typeof(BasePostParser<,>), typeof(BaseSaver<>) + typeof(BaseCrawler<,>), typeof(CrawlFacade<,,,>), + typeof(PostParser<,>), typeof(BaseSaver<>) ]); builder.RegisterType(); builder.RegisterType(); diff --git a/c#/crawler/src/Tieba/Crawl/Crawler/ThreadLateCrawler.cs b/c#/crawler/src/Tieba/Crawl/Crawler/ThreadLateCrawler.cs index d663bfca..812e7fbe 100644 --- a/c#/crawler/src/Tieba/Crawl/Crawler/ThreadLateCrawler.cs +++ b/c#/crawler/src/Tieba/Crawl/Crawler/ThreadLateCrawler.cs @@ -83,7 +83,7 @@ public class ThreadLateCrawler( } } catch (Exception e) - { // below is similar with BaseCrawlFacade.SilenceException() + { // below is similar with CrawlFacade.SilenceException() e.Data["fid"] = fid; e.Data["tid"] = tid; e = e.ExtractInnerExceptionsData(); diff --git a/c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs similarity index 93% rename from c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs rename to c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs index 0edce0c5..277ef395 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs @@ -1,14 +1,14 @@ namespace tbm.Crawler.Tieba.Crawl.Facade; #pragma warning disable S3881 // "IDisposable" should be implemented correctly -public abstract class BaseCrawlFacade( +public abstract class CrawlFacade( #pragma warning restore S3881 // "IDisposable" should be implemented correctly BaseCrawler crawler, Fid fid, CrawlerLocks.LockId lockId, CrawlerLocks locks, - BasePostParser postParser, - Func, BasePostSaver> postSaverFactory, + PostParser postParser, + Func, PostSaver> postSaverFactory, Func, UserParser> userParserFactory, Func, UserSaver> userSaverFactory) : IDisposable @@ -25,7 +25,7 @@ public abstract class BaseCrawlFacade> + public required ILogger> Logger { private get; init; } public required CrawlerDbContext.New DbContextFactory { private get; init; } public required ClientRequesterTcs RequesterTcs { private get; init; } @@ -77,11 +77,11 @@ public virtual void Dispose() } } - public async Task> + public async Task> CrawlPageRange(Page startPage, Page endPage = Page.MaxValue, CancellationToken stoppingToken = default) { // cancel when startPage is already locked if (_lockingPages.Count != 0) ThrowHelper.ThrowInvalidOperationException( - "CrawlPageRange() can only be called once, a instance of BaseCrawlFacade shouldn't be reuse for other crawls."); + "CrawlPageRange() can only be called once, a instance of CrawlFacade shouldn't be reuse for other crawls."); var acquiredLocks = locks.AcquireRange(lockId, [startPage]); if (acquiredLocks.Count == 0) Logger.LogInformation( "Cannot crawl any page within the range [{}-{}] for lock type {}, id {} since they've already been locked", @@ -115,12 +115,12 @@ public virtual void Dispose() (IReadOnlyList pages, Func failureCountSelector, CancellationToken stoppingToken = default) { if (_lockingPages.Count != 0) ThrowHelper.ThrowInvalidOperationException( - "RetryPages() can only be called once, a instance of BaseCrawlFacade shouldn't be reuse for other crawls."); + "RetryPages() can only be called once, a instance of CrawlFacade shouldn't be reuse for other crawls."); await CrawlPages(pages, failureCountSelector, stoppingToken); return SaveCrawled(stoppingToken); } - public BaseCrawlFacade + public CrawlFacade AddExceptionHandler(ExceptionHandler handler) { _exceptionHandler += handler; diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs index 31cc097f..d34a5a58 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs @@ -11,7 +11,7 @@ public class ReplyCrawlFacade( UserSaver.New userSaverFactory, CrawlerDbContext.New dbContextFactory, SonicPusher sonicPusher) - : BaseCrawlFacade( + : CrawlFacade( crawlerFactory(fid, tid), fid, new(fid, tid), locks[CrawlerLocks.Type.Reply], postParser, postSaverFactory.Invoke, userParserFactory.Invoke, userSaverFactory.Invoke) diff --git a/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs index 11c4a3be..cefb884c 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs @@ -11,7 +11,7 @@ public class SubReplyCrawlFacade( UserParser.New userParserFactory, UserSaver.New userSaverFactory, SonicPusher sonicPusher) - : BaseCrawlFacade( + : CrawlFacade( crawlerFactory(tid, pid), fid, new(fid, tid, pid), locks[CrawlerLocks.Type.SubReply], postParser, postSaverFactory.Invoke, userParserFactory.Invoke, userSaverFactory.Invoke) diff --git a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs index 76dad0c3..7cf23405 100644 --- a/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs +++ b/c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs @@ -9,7 +9,7 @@ public class ThreadCrawlFacade( ThreadSaver.New postSaverFactory, UserParser.New userParserFactory, UserSaver.New userSaverFactory) - : BaseCrawlFacade( + : CrawlFacade( crawlerFactory(forumName), fid, new(fid), locks[CrawlerLocks.Type.Thread], postParser, postSaverFactory.Invoke, userParserFactory.Invoke, userSaverFactory.Invoke) diff --git a/c#/crawler/src/Tieba/Crawl/Parser/Post/BasePostParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/Post/PostParser.cs similarity index 96% rename from c#/crawler/src/Tieba/Crawl/Parser/Post/BasePostParser.cs rename to c#/crawler/src/Tieba/Crawl/Parser/Post/PostParser.cs index a69a125f..83a99179 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/Post/BasePostParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/Post/PostParser.cs @@ -1,6 +1,6 @@ namespace tbm.Crawler.Tieba.Crawl.Parser.Post; -public abstract class BasePostParser +public abstract class PostParser where TPost : BasePost where TPostProtoBuf : class, IMessage { diff --git a/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs index d75b654d..2f55e79c 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/Post/ReplyParser.cs @@ -1,7 +1,7 @@ namespace tbm.Crawler.Tieba.Crawl.Parser; public partial class ReplyParser(ILogger logger) - : BasePostParser + : PostParser { // length with 24 char is only appeared in legacy replies [GeneratedRegex("^(?:[0-9a-f]{40}|[0-9a-f]{24})$", RegexOptions.Compiled, matchTimeoutMilliseconds: 100)] diff --git a/c#/crawler/src/Tieba/Crawl/Parser/Post/SubReplyParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/Post/SubReplyParser.cs index 8e35e514..6192fcc4 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/Post/SubReplyParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/Post/SubReplyParser.cs @@ -1,6 +1,6 @@ namespace tbm.Crawler.Tieba.Crawl.Parser.Post; -public class SubReplyParser : BasePostParser +public class SubReplyParser : PostParser { protected override PostId PostIdSelector(SubReplyPost post) => post.Spid; diff --git a/c#/crawler/src/Tieba/Crawl/Parser/Post/ThreadParser.cs b/c#/crawler/src/Tieba/Crawl/Parser/Post/ThreadParser.cs index 5a42e05d..ddaa4523 100644 --- a/c#/crawler/src/Tieba/Crawl/Parser/Post/ThreadParser.cs +++ b/c#/crawler/src/Tieba/Crawl/Parser/Post/ThreadParser.cs @@ -1,6 +1,6 @@ namespace tbm.Crawler.Tieba.Crawl.Parser.Post; -public class ThreadParser : BasePostParser +public class ThreadParser : PostParser { protected override PostId PostIdSelector(ThreadPost post) => post.Tid; diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/BasePostSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs similarity index 94% rename from c#/crawler/src/Tieba/Crawl/Saver/Post/BasePostSaver.cs rename to c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs index 40537786..146514a2 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/BasePostSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs @@ -2,8 +2,8 @@ namespace tbm.Crawler.Tieba.Crawl.Saver.Post; -public abstract class BasePostSaver( - ILogger> logger, +public abstract class PostSaver( + ILogger> logger, ConcurrentDictionary posts, AuthorRevisionSaver.New authorRevisionSaverFactory, PostType currentPostType) diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs index 663e258f..e9e3661f 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs @@ -7,7 +7,7 @@ public partial class ReplySaver( ILogger logger, ConcurrentDictionary posts, AuthorRevisionSaver.New authorRevisionSaverFactory) - : BasePostSaver( + : PostSaver( logger, posts, authorRevisionSaverFactory, PostType.Reply) { public delegate ReplySaver New(ConcurrentDictionary posts); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs index c16e8b47..209011f9 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs @@ -6,7 +6,7 @@ public class SubReplySaver( ILogger logger, ConcurrentDictionary posts, AuthorRevisionSaver.New authorRevisionSaverFactory) - : BasePostSaver( + : PostSaver( logger, posts, authorRevisionSaverFactory, PostType.SubReply) { public delegate SubReplySaver New(ConcurrentDictionary posts); diff --git a/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs b/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs index a941918f..639e9890 100644 --- a/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs +++ b/c#/crawler/src/Tieba/Crawl/Saver/Post/ThreadSaver.cs @@ -6,7 +6,7 @@ public class ThreadSaver( ILogger logger, ConcurrentDictionary posts, AuthorRevisionSaver.New authorRevisionSaverFactory) - : BasePostSaver( + : PostSaver( logger, posts, authorRevisionSaverFactory, PostType.Thread) { public delegate ThreadSaver New(ConcurrentDictionary posts);