Skip to content

Commit

Permalink
* convert interface IPost to abstract class BasePost
Browse files Browse the repository at this point in the history
* convert interface `IPostContent` to abstract class `PostContent`
* convert interface `IPostWithAuthorExpGrade` to abstract class `PostWithAuthorExpGrade`
@ Post

* convert interface `IRevision` to abstract class `BaseRevisionWithSplitting`
@ Splitting
+ abstract class `ForumScopedRevision`
@ Revision

* convert interface `ITimestampedEntity` to abstract class `TimestampedEntity`
@ Db
@ crawler

* convert interface `IEmbedded` to abstract class `Embedded`
- interface `IImageMetadata` in favor of `tbm.Shared.Db.EntityWithImageId.AsKey`
@ ImageMetadata.cs

+ abstract class `ImageWithFrameIndex`
@ Db
@ imagePipeline

+ abstract class `EntityWithImageId` and its nested abstract class `AsKey`
@ Db
@ shared
@ c#
  • Loading branch information
n0099 committed May 10, 2024
1 parent be4ecf3 commit f8d6e6f
Show file tree
Hide file tree
Showing 36 changed files with 94 additions and 139 deletions.
4 changes: 2 additions & 2 deletions c#/crawler/src/Db/CrawlerDbContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,13 @@ public class CrawlerDbContext(Fid fid) : TbmDbContext<CrawlerDbContext.ModelCach
public void TimestampingEntities() =>

// https://www.entityframeworktutorial.net/faq/set-created-and-modified-date-in-efcore.aspx
ChangeTracker.Entries<ITimestampedEntity>().ForEach(e =>
ChangeTracker.Entries<TimestampedEntity>().ForEach(e =>
{
Helper.GetNowTimestamp(out var now);
var originalEntityState = e.State; // copy e.State since it might change after any prop value updated
var createdAtProp = e.Property(ie => ie.CreatedAt);
var updatedAtProp = e.Property(ie => ie.UpdatedAt);
var lastSeenAtProp = e.Entity is IPost ? e.Property(ie => ((IPost)ie).LastSeenAt) : null;
var lastSeenAtProp = e.Entity is BasePost ? e.Property(ie => ((BasePost)ie).LastSeenAt) : null;
// ReSharper disable once SwitchStatementMissingSomeEnumCasesNoDefault
switch (originalEntityState)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
// ReSharper disable UnusedMemberInSuper.Global
namespace tbm.Crawler.Db.Post;

public interface IPost : ITimestampedEntity, ICloneable
public abstract class BasePost : TimestampedEntity, ICloneable
{
public ulong Tid { get; set; }
public long AuthorUid { get; set; }
public uint? LastSeenAt { get; set; }
public abstract object Clone();
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace tbm.Crawler.Db.Post;

public interface IPostContent
public abstract class PostContent : RowVersionedEntity
{
public byte[]? ProtoBufBytes { get; set; }
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// ReSharper disable UnusedMemberInSuper.Global
namespace tbm.Crawler.Db.Post;

public interface IPostWithAuthorExpGrade : IPost
public abstract class PostWithAuthorExpGrade : BasePost
{
[NotMapped] public byte AuthorExpGrade { get; set; }
}
3 changes: 1 addition & 2 deletions c#/crawler/src/Db/Post/ReplyContent.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
// ReSharper disable PropertyCanBeMadeInitOnly.Global
namespace tbm.Crawler.Db.Post;

public class ReplyContent : RowVersionedEntity, IPostContent
public class ReplyContent : PostContent
{
[Key] public ulong Pid { get; set; }
public byte[]? ProtoBufBytes { get; set; }
}
10 changes: 2 additions & 8 deletions c#/crawler/src/Db/Post/ReplyPost.cs
Original file line number Diff line number Diff line change
@@ -1,18 +1,15 @@
// ReSharper disable PropertyCanBeMadeInitOnly.Global
namespace tbm.Crawler.Db.Post;

public class ReplyPost : RowVersionedEntity, IPostWithAuthorExpGrade
public class ReplyPost : PostWithAuthorExpGrade
{
public ulong Tid { get; set; }
[Key] public ulong Pid { get; set; }
public uint Floor { get; set; }
[NotMapped] public byte[]? Content { get; set; }

[JsonConverter(typeof(ProtoBufRepeatedFieldJsonConverter<Content>))]
[NotMapped]
public required RepeatedField<Content> OriginalContents { get; set; }
public long AuthorUid { get; set; }
[NotMapped] public byte AuthorExpGrade { get; set; }
public uint? SubReplyCount { get; set; }
public uint PostedAt { get; set; }
public byte? IsFold { get; set; }
Expand All @@ -21,9 +18,6 @@ public class ReplyPost : RowVersionedEntity, IPostWithAuthorExpGrade
public byte[]? Geolocation { get; set; }
public uint? SignatureId { get; set; }
[NotMapped] public byte[]? Signature { get; set; }
public uint CreatedAt { get; set; }
public uint? UpdatedAt { get; set; }
public uint? LastSeenAt { get; set; }

public object Clone() => MemberwiseClone();
public override object Clone() => MemberwiseClone();
}
3 changes: 1 addition & 2 deletions c#/crawler/src/Db/Post/SubReplyContent.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
// ReSharper disable PropertyCanBeMadeInitOnly.Global
namespace tbm.Crawler.Db.Post;

public class SubReplyContent : RowVersionedEntity, IPostContent
public class SubReplyContent : PostContent
{
[Key] public ulong Spid { get; set; }
public byte[]? ProtoBufBytes { get; set; }
}
10 changes: 2 additions & 8 deletions c#/crawler/src/Db/Post/SubReplyPost.cs
Original file line number Diff line number Diff line change
@@ -1,24 +1,18 @@
// ReSharper disable PropertyCanBeMadeInitOnly.Global
namespace tbm.Crawler.Db.Post;

public class SubReplyPost : RowVersionedEntity, IPostWithAuthorExpGrade
public class SubReplyPost : PostWithAuthorExpGrade
{
public ulong Tid { get; set; }
public ulong Pid { get; set; }
[Key] public ulong Spid { get; set; }
[NotMapped] public byte[]? Content { get; set; }

[JsonConverter(typeof(ProtoBufRepeatedFieldJsonConverter<Content>))]
[NotMapped]
public required RepeatedField<Content> OriginalContents { get; set; }
public long AuthorUid { get; set; }
[NotMapped] public byte AuthorExpGrade { get; set; }
public uint PostedAt { get; set; }
public int? AgreeCount { get; set; }
public int? DisagreeCount { get; set; }
public uint CreatedAt { get; set; }
public uint? UpdatedAt { get; set; }
public uint? LastSeenAt { get; set; }

public object Clone() => MemberwiseClone();
public override object Clone() => MemberwiseClone();
}
10 changes: 3 additions & 7 deletions c#/crawler/src/Db/Post/ThreadPost.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// ReSharper disable PropertyCanBeMadeInitOnly.Global
namespace tbm.Crawler.Db.Post;

public class ThreadPost : RowVersionedEntity, IPost
public class ThreadPost : BasePost
{
[Key] public ulong Tid { get; set; }
[Key] public new ulong Tid { get; set; }
[NotMapped] public ulong? FirstReplyPid { get; set; }

[JsonConverter(typeof(ProtoBufRepeatedFieldJsonConverter<Abstract>))]
Expand All @@ -14,7 +14,6 @@ public class ThreadPost : RowVersionedEntity, IPost
public string? TopicType { get; set; }
public byte? IsGood { get; set; }
public required string Title { get; set; }
public long AuthorUid { get; set; }
public string? AuthorPhoneType { get; set; }
public uint PostedAt { get; set; }
public uint LatestReplyPostedAt { get; set; }
Expand All @@ -26,9 +25,6 @@ public class ThreadPost : RowVersionedEntity, IPost
public int? DisagreeCount { get; set; }
public byte[]? Zan { get; set; }
public byte[]? Geolocation { get; set; }
public uint CreatedAt { get; set; }
public uint? UpdatedAt { get; set; }
public uint? LastSeenAt { get; set; }

public object Clone() => MemberwiseClone();
public override object Clone() => MemberwiseClone();
}
4 changes: 1 addition & 3 deletions c#/crawler/src/Db/Revision/AuthorRevision.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,8 @@
// ReSharper disable UnusedAutoPropertyAccessor.Global
namespace tbm.Crawler.Db.Revision;

public abstract class AuthorRevision : RowVersionedEntity
public abstract class AuthorRevision : ForumScopedRevision
{
public uint DiscoveredAt { get; set; }
public uint Fid { get; set; }
public long Uid { get; set; }
public required PostType TriggeredBy { get; set; }
}
4 changes: 1 addition & 3 deletions c#/crawler/src/Db/Revision/ForumModeratorRevision.cs
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
// ReSharper disable PropertyCanBeMadeInitOnly.Global
namespace tbm.Crawler.Db.Revision;

public class ForumModeratorRevision : RowVersionedEntity
public class ForumModeratorRevision : ForumScopedRevision
{
public uint DiscoveredAt { get; set; }
public uint Fid { get; set; }
public required string Portrait { get; set; }
public required string ModeratorTypes { get; set; }
}
7 changes: 7 additions & 0 deletions c#/crawler/src/Db/Revision/ForumScopedRevision.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace tbm.Crawler.Db.Revision;

public abstract class ForumScopedRevision : RowVersionedEntity
{
public uint DiscoveredAt { get; set; }
public uint Fid { get; set; }
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// ReSharper disable UnusedMemberInSuper.Global
namespace tbm.Crawler.Db.Revision.Splitting;

public interface IRevision
public abstract class BaseRevisionWithSplitting : RowVersionedEntity
{
public uint TakenAt { get; set; }
public ushort? NullFieldsBitMask { get; set; }
public bool IsAllFieldsIsNullExceptSplit();
public abstract bool IsAllFieldsIsNullExceptSplit();
}
10 changes: 3 additions & 7 deletions c#/crawler/src/Db/Revision/Splitting/RevisionWithSplitting.cs
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
namespace tbm.Crawler.Db.Revision.Splitting;

public abstract class RevisionWithSplitting<TBaseRevision> : RowVersionedEntity, IRevision
where TBaseRevision : class, IRevision
public abstract class RevisionWithSplitting<TBaseRevision> : BaseRevisionWithSplitting
where TBaseRevision : BaseRevisionWithSplitting
{
private readonly Dictionary<Type, TBaseRevision> _splitEntities = [];

public uint TakenAt { get; set; }
public ushort? NullFieldsBitMask { get; set; }
public IReadOnlyDictionary<Type, TBaseRevision> SplitEntities => _splitEntities;

public virtual bool IsAllFieldsIsNullExceptSplit() => throw new NotSupportedException();
public override bool IsAllFieldsIsNullExceptSplit() => throw new NotSupportedException();

protected TValue? GetSplitEntityValue<TSplitEntity, TValue>
(Func<TSplitEntity, TValue?> valueSelector)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace tbm.Crawler.Db;

public interface ITimestampedEntity
public abstract class TimestampedEntity : RowVersionedEntity
{
public uint CreatedAt { get; set; }
public uint? UpdatedAt { get; set; }
Expand Down
4 changes: 1 addition & 3 deletions c#/crawler/src/Db/User.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace tbm.Crawler.Db;

public class User : RowVersionedEntity, ITimestampedEntity
public class User : TimestampedEntity
{
[Key] public long Uid { get; set; }
public string? Name { get; set; }
Expand All @@ -11,8 +11,6 @@ public class User : RowVersionedEntity, ITimestampedEntity
public string? FansNickname { get; set; }
public byte[]? Icon { get; set; }
public string? IpGeolocation { get; set; }
public uint CreatedAt { get; set; }
public uint? UpdatedAt { get; set; }

public static User CreateLatestReplier(long uid, string? name, string? displayName) =>
new() {Uid = uid, Name = name, DisplayName = displayName, Portrait = ""};
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Facade/BaseCrawlFacade.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ public abstract class BaseCrawlFacade<TPost, TBaseRevision, TResponse, TPostProt
Func<ConcurrentDictionary<Uid, User>, UserParser> userParserFactory,
Func<ConcurrentDictionary<Uid, User>, UserSaver> userSaverFactory)
: IDisposable
where TPost : class, IPost
where TBaseRevision : class, IRevision
where TPost : BasePost
where TBaseRevision : BaseRevisionWithSplitting
where TResponse : class, IMessage<TResponse>
where TPostProtoBuf : class, IMessage<TPostProtoBuf>
{
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Parser/Post/BasePostParser.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
namespace tbm.Crawler.Tieba.Crawl.Parser.Post;

public abstract class BasePostParser<TPost, TPostProtoBuf>
where TPost : class, IPost
where TPost : BasePost
where TPostProtoBuf : class, IMessage<TPostProtoBuf>
{
public void Parse(
Expand All @@ -18,7 +18,7 @@ public abstract class BasePostParser<TPost, TPostProtoBuf>
outPosts = ParseInternal(inPosts, nullableUsers).ToDictionary(PostIdSelector, post => post);
if (outPosts.Values.Any(p => p.AuthorUid == 0))
throw new TiebaException(shouldRetry: true,
"Value of IPost.AuthorUid is the protoBuf default value 0.");
"Value of BasePost.AuthorUid is the protoBuf default value 0.");

var users = new List<TbClient.User>(30);
users.AddRange(nullableUsers.OfType<TbClient.User>()
Expand Down
4 changes: 2 additions & 2 deletions c#/crawler/src/Tieba/Crawl/Saver/AuthorRevisionSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public class AuthorRevisionSaver(PostType triggeredByPostType)

public Action SaveAuthorExpGradeRevisions<TPostWithAuthorExpGrade>
(CrawlerDbContext db, IReadOnlyCollection<TPostWithAuthorExpGrade> posts)
where TPostWithAuthorExpGrade : class, IPost, IPostWithAuthorExpGrade
where TPostWithAuthorExpGrade : PostWithAuthorExpGrade
{
SaveAuthorRevisions(db, posts, AuthorExpGradeLocks,
db.AuthorExpGradeRevisions,
Expand Down Expand Up @@ -46,7 +46,7 @@ public class AuthorRevisionSaver(PostType triggeredByPostType)
Func<TValue?, TValue?, bool> isValueChangedPredicate,
Expression<Func<TRevision, LatestAuthorRevisionProjection<TValue>>> latestRevisionProjectionFactory,
Func<(Uid Uid, TValue? Value, Time DiscoveredAt), TRevision> revisionFactory)
where TPost : class, IPost
where TPost : BasePost
where TRevision : AuthorRevision
{
Helper.GetNowTimestamp(out var now);
Expand Down
10 changes: 5 additions & 5 deletions c#/crawler/src/Tieba/Crawl/Saver/BaseSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ public abstract class BaseSaver<TBaseRevision>(ILogger<BaseSaver<TBaseRevision>>
#pragma warning disable S1939 // Inheritance list should not be redundant
: SaverWithRevision<TBaseRevision>, IFieldChangeIgnorance
#pragma warning restore S1939 // Inheritance list should not be redundant
where TBaseRevision : class, IRevision
where TBaseRevision : BaseRevisionWithSplitting
{
protected void SavePostsOrUsers<TPostOrUser, TRevision>(
CrawlerDbContext db,
Expand All @@ -15,7 +15,7 @@ public abstract class BaseSaver<TBaseRevision>(ILogger<BaseSaver<TBaseRevision>>
ILookup<bool, TPostOrUser> existingOrNewLookup,
Func<TPostOrUser, TPostOrUser> existingSelector)
where TPostOrUser : class
where TRevision : class, IRevision
where TRevision : BaseRevisionWithSplitting
{
db.Set<TPostOrUser>().AddRange(existingOrNewLookup[false]); // newly added
var newRevisions = existingOrNewLookup[true].Select(newPostOrUser =>
Expand All @@ -26,11 +26,11 @@ public abstract class BaseSaver<TBaseRevision>(ILogger<BaseSaver<TBaseRevision>>
// this will mutate postOrUserInTracking which is referenced by entry
entry.CurrentValues.SetValues(newPostOrUser);
bool IsTimestampingFieldName(string name) => name is nameof(IPost.LastSeenAt)
or nameof(ITimestampedEntity.CreatedAt) or nameof(ITimestampedEntity.UpdatedAt);
bool IsTimestampingFieldName(string name) => name is nameof(BasePost.LastSeenAt)
or nameof(TimestampedEntity.CreatedAt) or nameof(TimestampedEntity.UpdatedAt);
// rollback changes that overwrite original values with the default value 0 or null
// for all fields of ITimestampedEntity and IPost.LastSeenAt
// for all fields of TimestampedEntity and BasePost.LastSeenAt
// this will also affect the entity instance which postOrUserInTracking references to it
entry.Properties
.Where(prop => prop.IsModified && IsTimestampingFieldName(prop.Metadata.Name))
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/IFieldChangeIgnorance.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public partial interface IFieldChangeIgnorance
&& newValue is null && oldValue is not null) return true;
// possible rarely respond with the protoBuf default value 0
return propName == nameof(IPost.AuthorUid)
return propName == nameof(BasePost.AuthorUid)
&& newValue is 0L && oldValue is not null;
},
Revision: (whichPostType, propName, oldValue, _) =>
Expand Down
6 changes: 3 additions & 3 deletions c#/crawler/src/Tieba/Crawl/Saver/Post/BasePostSaver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ public abstract class BasePostSaver<TPost, TBaseRevision>(
AuthorRevisionSaver.New authorRevisionSaverFactory,
PostType currentPostType)
: BaseSaver<TBaseRevision>(logger)
where TPost : class, IPost
where TBaseRevision : class, IRevision
where TPost : BasePost
where TBaseRevision : BaseRevisionWithSplitting
{
protected delegate void PostSaveEventHandler();
[SuppressMessage("Design", "MA0046:Use EventHandler<T> to declare events")]
Expand All @@ -30,7 +30,7 @@ public virtual IFieldChangeIgnorance.FieldChangeIgnoranceDelegates
Func<TPost, PostId> postIdSelector,
Func<TPost, TRevision> revisionFactory,
ExpressionStarter<TPost> existingPostPredicate)
where TRevision : class, IRevision
where TRevision : BaseRevisionWithSplitting
{
var existingPostsKeyById = db.Set<TPost>()
.Where(existingPostPredicate).ToDictionary(postIdSelector);
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/SaverChangeSet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ public class SaverChangeSet<TPost>(
IReadOnlyCollection<TPost> existingBefore,
ICollection<TPost> existingAfterAndNewlyAdded,
Func<TPost, PostId> postIdSelector)
where TPost : class, IPost
where TPost : BasePost
{
public IReadOnlyCollection<(TPost Before, TPost After)> Existing { get; } = existingBefore
.OrderBy(postIdSelector)
Expand Down
2 changes: 1 addition & 1 deletion c#/crawler/src/Tieba/Crawl/Saver/SaverWithRevision.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
namespace tbm.Crawler.Tieba.Crawl.Saver;

public abstract class SaverWithRevision<TBaseRevision> : IRevisionProperties
where TBaseRevision : class, IRevision
where TBaseRevision : BaseRevisionWithSplitting
{
protected delegate void RevisionUpsertDelegate(CrawlerDbContext db, IEnumerable<TBaseRevision> revision);

Expand Down
2 changes: 1 addition & 1 deletion c#/imagePipeline/src/Consumer/MetadataConsumer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public MetadataConsumer
TImageSharpProfile? profile,
Func<TImageSharpProfile, byte[]?> rawBytesSelector)
where TImageSharpProfile : class
where TEmbeddedMetadata : class, ImageMetadata.IEmbedded, new()
where TEmbeddedMetadata : ImageMetadata.Embedded, new()
{
if (profile == null) return null;
var rawBytes = rawBytesSelector(profile); // will be null when param profile is null
Expand Down
3 changes: 1 addition & 2 deletions c#/imagePipeline/src/Db/ImageFailed.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@

namespace tbm.ImagePipeline.Db;

public class ImageFailed : RowVersionedEntity
public class ImageFailed : EntityWithImageId
{
[Key] [DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public uint Id { get; set; }
public uint ImageId { get; set; }
public required string Exception { get; set; }
}
Loading

0 comments on commit f8d6e6f

Please sign in to comment.