Skip to content
This repository was archived by the owner on Dec 2, 2022. It is now read-only.

1.0.x #20

Merged
merged 3 commits into from
Aug 13, 2021
Merged

1.0.x #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
198c2b4
Email validity check on signup
DazornSama Aug 13, 2021
4f201ec
Removed read-only fields from MongoDB
DazornSama Aug 13, 2021
d9b3b49
Implemented #17
DazornSama Aug 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Commons/User.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ public class User : IModel

[JsonPropertyName("has_anilist")]
[JsonProperty(PropertyName = "has_anilist")]
[BsonIgnore]
public bool? HasAnilist { get; set; } = null;

[BsonElement("mal_id")]
Expand All @@ -94,6 +95,7 @@ public class User : IModel

[JsonPropertyName("has_mal")]
[JsonProperty(PropertyName = "has_mal")]
[BsonIgnore]
public bool? HasMyAnimeList { get; set; } = null;

public void CalcDerivedFields()
Expand Down
2 changes: 2 additions & 0 deletions SyncService/Models/AnimeMatching.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,7 @@ public class AnimeMatching
public int Score { get; set; }
public string Path { get; set; }
public List<EpisodeMatching> Episodes { get; set; } = new List<EpisodeMatching>();
public string SourceVariant { get; set; }
public AnimeMatching Linked { get; set; }
}
}
89 changes: 53 additions & 36 deletions SyncService/Models/IWebsiteScraper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ public abstract class IWebsiteScraper
protected Thread Thread { get; private set; }
public bool Working { get; private set; } = false;

protected List<EpisodeMatching> EpisodeMatchings = new List<EpisodeMatching>();

public IWebsiteScraper(WebsiteScraperService service)
{
this.Service = service;
Expand Down Expand Up @@ -174,43 +176,16 @@ private async Task run()

try
{
for (int i = 1; i <= _anime.EpisodesCount; i++)
AnimeMatching linked = matching.Linked;

await getEpisodes(browser, matching);

while (linked != null)
{
browser = await ProxyHelper.Instance.GetBrowser();
try
{
using (Page webPage = await ProxyHelper.Instance.GetBestProxy(browser, this.Website.CanBlockRequests))
{
EpisodeMatching episode = await this.GetEpisode(webPage, matching, i);

if (episode != null)
{
Episode ep = new Episode()
{
AnimeID = _anime.Id,
Source = this.Website.Name,
Number = episode.Number,
Title = episode.Title,
Video = episode.Source,
Locale = this.Website.Localization
};

if (this._episodeCollection.Exists(ref ep))
{
this._episodeCollection.Edit(ref ep);
}
else
{
this._episodeCollection.Add(ref ep);
}
}
}
}
catch { }
finally
{
await browser.CloseAsync();
}
EpisodeMatchings.Clear();

await getEpisodes(browser, linked);
linked = linked.Linked;
}
}
catch
Expand Down Expand Up @@ -264,6 +239,48 @@ private async Task run()
}
}

private async Task getEpisodes(Browser browser, AnimeMatching matching)
{
for (int i = 1; i <= _anime.EpisodesCount; i++)
{
browser = await ProxyHelper.Instance.GetBrowser();
try
{
using (Page webPage = await ProxyHelper.Instance.GetBestProxy(browser, this.Website.CanBlockRequests))
{
EpisodeMatching episode = await this.GetEpisode(webPage, matching, i);

if (episode != null)
{
Episode ep = new Episode()
{
AnimeID = _anime.Id,
Source = $"{this.Website.Name}{matching.SourceVariant}",
Number = episode.Number,
Title = episode.Title,
Video = episode.Source,
Locale = this.Website.Localization
};

if (this._episodeCollection.Exists(ref ep))
{
this._episodeCollection.Edit(ref ep);
}
else
{
this._episodeCollection.Add(ref ep);
}
}
}
}
catch { }
finally
{
await browser.CloseAsync();
}
}
}

private bool animeNeedWork(bool forceReload = false)
{
if (forceReload)
Expand Down
10 changes: 4 additions & 6 deletions SyncService/Models/WebsiteScrapers/AnimeworldScraper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,14 @@ public AnimeworldScraper(WebsiteScraperService service) : base(service)

protected override Type WebsiteType => typeof(AnimeworldScraper);

private List<EpisodeMatching> episodesMatchings = new List<EpisodeMatching>();

protected override async Task<AnimeMatching> GetMatching(Page webPage, string animeTitle)
{
AnimeMatching matching = null;

string url = $"{this.Website.SiteUrl}search?keyword={animeTitle}";
await ProxyHelper.NavigateAsync(webPage, url);

episodesMatchings.Clear();
EpisodeMatchings.Clear();

await webPage.WaitForSelectorAsync(".film-list", new WaitForSelectorOptions()
{
Expand Down Expand Up @@ -62,7 +60,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
{
string url;

if (episodesMatchings.Count == 0)
if (EpisodeMatchings.Count == 0)
{
url = this.Website.SiteUrl.Substring(0, this.Website.SiteUrl.Length - 1);
url = $"{url}{matching.Path}";
Expand Down Expand Up @@ -94,7 +92,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
string path = await info.EvaluateFunctionAsync<string>("e => e.getAttribute('href')");
string title = (await info.EvaluateFunctionAsync<string>("e => e.innerText")).Trim();

episodesMatchings.Add(new EpisodeMatching()
EpisodeMatchings.Add(new EpisodeMatching()
{
Path = path,
Title = title
Expand All @@ -105,7 +103,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
}
}

EpisodeMatching episode = episodesMatchings[number - 1];
EpisodeMatching episode = EpisodeMatchings[number - 1];

if (episode != null)
{
Expand Down
10 changes: 4 additions & 6 deletions SyncService/Models/WebsiteScrapers/DreamsubScraper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
{
AnimeMatching matching = null;

episodesMatchings.Clear();
EpisodeMatchings.Clear();

string url = $"{this.Website.SiteUrl}/search/?q={animeTitle}";
await ProxyHelper.NavigateAsync(webPage, url);
Expand Down Expand Up @@ -56,13 +56,11 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
return null;
}

private List<EpisodeMatching> episodesMatchings = new List<EpisodeMatching>();

protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMatching matching, int number)
{
string url;

if(episodesMatchings.Count == 0)
if(EpisodeMatchings.Count == 0)
{
url = $"{this.Website.SiteUrl}{matching.Path}";
await ProxyHelper.NavigateAsync(webPage, url);
Expand All @@ -83,15 +81,15 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
string path = await info.EvaluateFunctionAsync<string>("e => e.getAttribute('href')");
string title = (await info.EvaluateFunctionAsync<string>("e => e.innerText")).Trim().Split(": ")[1];

episodesMatchings.Add(new EpisodeMatching()
EpisodeMatchings.Add(new EpisodeMatching()
{
Path = path,
Title = title
});
}
}

EpisodeMatching episode = episodesMatchings[number - 1];
EpisodeMatching episode = EpisodeMatchings[number - 1];

if(episode != null)
{
Expand Down
19 changes: 12 additions & 7 deletions SyncService/Models/WebsiteScrapers/GogoanimeScraper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ public GogoanimeScraper(WebsiteScraperService service) : base(service)

protected override Type WebsiteType => typeof(GogoanimeScraper);

private List<EpisodeMatching> episodesMatchings = new List<EpisodeMatching>();

protected override async Task<AnimeMatching> GetMatching(Page webPage, string animeTitle)
{
AnimeMatching matching = null;
Expand All @@ -44,7 +42,7 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
new WebsiteCollection().Edit(ref website);
}

episodesMatchings.Clear();
EpisodeMatchings.Clear();

await webPage.WaitForSelectorAsync(".last_episodes", new WaitForSelectorOptions()
{
Expand All @@ -65,6 +63,13 @@ protected override async Task<AnimeMatching> GetMatching(Page webPage, string an
ElementHandle path = await title.QuerySelectorAsync("a");
matching.Path = (await path.EvaluateFunctionAsync<string>("e => e.getAttribute('href')")).Trim();

matching.Linked = new AnimeMatching()
{
Title = matching.Title,
Path = $"{matching.Path}-dub",
SourceVariant = "_dub"
};

return matching;
}
}
Expand All @@ -76,7 +81,7 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
{
string url;

if (episodesMatchings.Count == 0)
if (EpisodeMatchings.Count == 0)
{
url = this.Website.SiteUrl.Substring(0, this.Website.SiteUrl.Length - 1);
url = $"{url}{matching.Path}";
Expand Down Expand Up @@ -114,16 +119,16 @@ protected override async Task<EpisodeMatching> GetEpisode(Page webPage, AnimeMat
string path = await info.EvaluateFunctionAsync<string>("e => e.getAttribute('href')");
string title = (await info.QuerySelectorAsync(".name").EvaluateFunctionAsync<string>("e => e.innerText")).Trim();

episodesMatchings.Add(new EpisodeMatching()
EpisodeMatchings.Add(new EpisodeMatching()
{
Path = path,
Title = title
});
}
episodesMatchings.Reverse();
EpisodeMatchings.Reverse();
}

EpisodeMatching episode = episodesMatchings[number - 1];
EpisodeMatching episode = EpisodeMatchings[number - 1];

if (episode != null)
{
Expand Down
8 changes: 8 additions & 0 deletions WebAPI/Controllers/UserController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Net;
using System.Net.Http;
using System.Net.Mail;
Expand Down Expand Up @@ -180,6 +181,13 @@ public async Task<APIResponse> Create(string g_recaptcha_response, [FromBody] Us
"Please provide a valid password");
}

if(!new EmailAddressAttribute().IsValid(model.Email))
{
throw new APIException(HttpStatusCode.UnprocessableEntity,
"Email not valid",
"Please provide a valid email");
}

List<User> collisions = new List<User>();

collisions.AddRange(this._userCollection.GetList(new UserFilter()
Expand Down