Skip to content

Commit

Permalink
* rename method PostParseHook() to OnPostParse()
Browse files Browse the repository at this point in the history
* rename method `BeforeCommitSaveHook()` to `OnBeforeCommitSave()`
* rename method `PostCommitSaveHook()` to `OnPostCommitSave()`
@ CrawlFacade.cs

* rename method `TriggerPostSave()` to `OnPostSave()` @ IPostSaver.cs
* rename delegate and prop `PostSaveHook(s)` to `PostSaveHandler(s)` @ PostSaver.cs

* rename method `PostSaveHook()` to `OnPostSave()` @ UserSaver.cs
@ c#/crawler
  • Loading branch information
n0099 committed May 11, 2024
1 parent 00b8e91 commit 074d477
Show file tree
Hide file tree
Showing 11 changed files with 30 additions and 27 deletions.
23 changes: 13 additions & 10 deletions c#/crawler/src/Tieba/Crawl/Facade/CrawlFacade.cs
Expand Up @@ -52,13 +52,13 @@ public virtual void Dispose()
var userSaver = userSaverFactory(_users);
userSaver.Save(db, postSaver.CurrentPostType, postSaver.UserFieldChangeIgnorance);

BeforeCommitSaveHook(db, userSaver);
OnBeforeCommitSave(db, userSaver);
try
{
db.TimestampingEntities();
_ = db.SaveChanges();
transaction.Commit();
if (savedPosts != null) PostCommitSaveHook(savedPosts, stoppingToken);
if (savedPosts != null) OnPostCommitSave(savedPosts, stoppingToken);
return savedPosts;
}
catch (DbUpdateConcurrencyException e)
Expand All @@ -67,8 +67,8 @@ public virtual void Dispose()
}
finally
{
postSaver.TriggerPostSave();
userSaver.PostSaveHook();
postSaver.OnPostSave();
userSaver.OnPostSave();
}
}
}
Expand Down Expand Up @@ -125,12 +125,12 @@ public ICrawlFacade<TPost> AddExceptionHandler(ICrawlFacade<TPost>.ExceptionHand
}

protected virtual void ThrowIfEmptyUsersEmbedInPosts() { }
protected virtual void PostParseHook(
protected virtual void OnPostParse(
TResponse response,
CrawlRequestFlag flag,
IReadOnlyDictionary<PostId, TPost> parsedPostsInResponse) { }
protected virtual void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver userSaver) { }
protected virtual void PostCommitSaveHook(
protected virtual void OnBeforeCommitSave(CrawlerDbContext db, UserSaver userSaver) { }
protected virtual void OnPostCommitSave(
SaverChangeSet<TPost> savedPosts,
CancellationToken stoppingToken = default) { }

Expand All @@ -145,7 +145,7 @@ private void ValidateThenParse(BaseCrawler<TResponse, TPostProtoBuf>.Response re
if (postsEmbeddedUsers.Count == 0 && postsInResponse.Count != 0) ThrowIfEmptyUsersEmbedInPosts();
if (postsEmbeddedUsers.Count != 0) UserParser.Parse(postsEmbeddedUsers);
}
PostParseHook(response, flag, parsedPostsInResponse);
OnPostParse(response, flag, parsedPostsInResponse);
}

private async Task CrawlPages(
Expand Down Expand Up @@ -173,8 +173,11 @@ private void ValidateThenParse(BaseCrawler<TResponse, TPostProtoBuf>.Response re
page, previousFailureCountSelector?.Invoke(page) ?? 0, stoppingToken)));
}

private async Task<bool> LogException
(Func<Task> payload, Page page, FailureCount previousFailureCount, CancellationToken stoppingToken = default)
private async Task<bool> LogException(
Func<Task> payload,
Page page,
FailureCount previousFailureCount,
CancellationToken stoppingToken = default)
{
try
{
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Facade/ReplyCrawlFacade.cs
Expand Up @@ -18,7 +18,7 @@ public class ReplyCrawlFacade(
{
public delegate ReplyCrawlFacade New(Fid fid, Tid tid);

protected override void PostParseHook(
protected override void OnPostParse(
ReplyResponse response,
CrawlRequestFlag flag,
IReadOnlyDictionary<PostId, ReplyPost> parsedPostsInResponse)
Expand All @@ -30,7 +30,7 @@ public class ReplyCrawlFacade(
if (data.Page.CurrentPage == 1) SaveParentThreadTitle(data.PostList);
}

protected override void PostCommitSaveHook(
protected override void OnPostCommitSave(
SaverChangeSet<ReplyPost> savedPosts,
CancellationToken stoppingToken = default) =>
sonicPusher.PushPostWithCancellationToken(savedPosts.NewlyAdded, Fid, "replies",
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Facade/SubReplyCrawlFacade.cs
Expand Up @@ -21,7 +21,7 @@ public class SubReplyCrawlFacade(
protected override void ThrowIfEmptyUsersEmbedInPosts() => throw new TiebaException(
$"User list in the response of sub reply request for fid {Fid}, tid {tid}, pid {pid} is empty.");

protected override void PostParseHook(
protected override void OnPostParse(
SubReplyResponse response,
CrawlRequestFlag flag,
IReadOnlyDictionary<PostId, SubReplyPost> parsedPostsInResponse)
Expand All @@ -34,7 +34,7 @@ public class SubReplyCrawlFacade(
UserParser.ResetUsersIcon();
}

protected override void PostCommitSaveHook(
protected override void OnPostCommitSave(
SaverChangeSet<SubReplyPost> savedPosts,
CancellationToken stoppingToken = default) =>
sonicPusher.PushPostWithCancellationToken(savedPosts.NewlyAdded, Fid, "subReplies",
Expand Down
Expand Up @@ -16,7 +16,7 @@ public class ThreadArchiveCrawlFacade(
{
public new delegate ThreadArchiveCrawlFacade New(Fid fid, string forumName);

protected override void PostParseHook(
protected override void OnPostParse(
ThreadResponse response,
CrawlRequestFlag flag,
IReadOnlyDictionary<PostId, ThreadPost> parsedPostsInResponse)
Expand Down
6 changes: 3 additions & 3 deletions c#/crawler/src/Tieba/Crawl/Facade/ThreadCrawlFacade.cs
Expand Up @@ -18,8 +18,8 @@ public class ThreadCrawlFacade(

public delegate ThreadCrawlFacade New(Fid fid, string forumName);

protected override void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver userSaver)
{ // BeforeCommitSaveHook() should get invoked after UserSaver.Save() by the base.SaveCrawled()
protected override void OnBeforeCommitSave(CrawlerDbContext db, UserSaver userSaver)
{ // OnBeforeCommitSave() should get invoked after UserSaver.Save() by the base.SaveCrawled()
// so only latest repliers that not exists in parsed users are being inserted
// note this will bypass user revision detection since not invoking BaseSaver.SavePostsOrUsers() but directly DbContext.AddRange()

Expand All @@ -42,7 +42,7 @@ protected override void BeforeCommitSaveHook(CrawlerDbContext db, UserSaver user
db.Users.AddRange(newLatestRepliersExceptLocked);
}

protected override void PostParseHook(
protected override void OnPostParse(
ThreadResponse response,
CrawlRequestFlag flag,
IReadOnlyDictionary<PostId, ThreadPost> parsedPostsInResponse)
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/IFieldChangeIgnorance.cs
Expand Up @@ -83,7 +83,7 @@ public partial interface IFieldChangeIgnorance
if (whichPostType == typeof(ThreadPost))
{
switch (propName)
{ // empty string from response has been updated by ReplyCrawlFacade.PostParseHook()
{ // empty string from response has been updated by ReplyCrawlFacade.OnPostParse()
case nameof(ThreadPost.Title) when oldValue is "":
// null values will be later set by tieba client 6.0.2 response at ThreadParser.ParseInternal()
case nameof(ThreadPost.LatestReplierUid) when oldValue is null:
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/IPostSaver.cs
Expand Up @@ -4,6 +4,6 @@ public interface IPostSaver<TPost> where TPost : BasePost
{
public IFieldChangeIgnorance.FieldChangeIgnoranceDelegates UserFieldChangeIgnorance { get; }
public PostType CurrentPostType { get; }
public void TriggerPostSave();
public void OnPostSave();
public SaverChangeSet<TPost> Save(CrawlerDbContext db);
}
6 changes: 3 additions & 3 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs
Expand Up @@ -11,16 +11,16 @@ public abstract class PostSaver<TPost, TBaseRevision>(
where TPost : BasePost
where TBaseRevision : BaseRevisionWithSplitting
{
protected delegate void PostSaveHook();
protected PostSaveHook PostSaveHooks { get; set; } = () => { };
protected delegate void PostSaveHandler();
protected PostSaveHandler PostSaveHandlers { get; set; } = () => { };

public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
UserFieldChangeIgnorance => throw new NotSupportedException();

Check failure on line 18 in c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs

View workflow job for this annotation

GitHub Actions / build (crawler)

Check failure on line 18 in c#/crawler/src/Tieba/Crawl/Saver/Post/PostSaver.cs

View workflow job for this annotation

GitHub Actions / build (crawler)

public PostType CurrentPostType { get; } = currentPostType;
protected ConcurrentDictionary<PostId, TPost> Posts { get; } = posts;
protected AuthorRevisionSaver AuthorRevisionSaver { get; } = authorRevisionSaverFactory(currentPostType);

public void TriggerPostSave() => PostSaveHooks();
public void OnPostSave() => PostSaveHandlers();
public abstract SaverChangeSet<TPost> Save(CrawlerDbContext db);

protected SaverChangeSet<TPost> Save<TRevision>(
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/ReplySaver.cs
Expand Up @@ -54,8 +54,8 @@ public override SaverChangeSet<ReplyPost> Save(CrawlerDbContext db)
db.ReplyContents.AddRange(changeSet.NewlyAdded
.Select(r => new ReplyContent {Pid = r.Pid, ProtoBufBytes = r.Content}));
SaveReplyContentImages(db, changeSet.NewlyAdded);
PostSaveHooks += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
PostSaveHooks += SaveReplySignatures(db, changeSet.AllAfter).Invoke;
PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
PostSaveHandlers += SaveReplySignatures(db, changeSet.AllAfter).Invoke;

return changeSet;
}
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/Post/SubReplySaver.cs
Expand Up @@ -49,7 +49,7 @@ public override SaverChangeSet<SubReplyPost> Save(CrawlerDbContext db)

db.SubReplyContents.AddRange(changeSet.NewlyAdded.Select(sr =>
new SubReplyContent {Spid = sr.Spid, ProtoBufBytes = sr.Content}));
PostSaveHooks += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;
PostSaveHandlers += AuthorRevisionSaver.SaveAuthorExpGradeRevisions(db, changeSet.AllAfter).Invoke;

return changeSet;
}
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/UserSaver.cs
Expand Up @@ -79,7 +79,7 @@ public IEnumerable<Uid> AcquireUidLocksForSave(IEnumerable<Uid> usersId)
}
}

public void PostSaveHook()
public void OnPostSave()
{
lock (UserIdLocks) if (_savedUsersId.Count != 0) UserIdLocks.ExceptWith(_savedUsersId);
}
Expand Down

0 comments on commit 074d477

Please sign in to comment.