up

pavlovtech · Apr 18, 2023 · 3357c67 · 3357c67
1 parent e4b2f10
commit 3357c67
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 6 deletions.
diff --git a/.../WebReaper.AzureFuncs/ExoScraperSpider.cs → ...s/WebReaper.AzureFuncs/WebReaperSpider.cs b/.../WebReaper.AzureFuncs/ExoScraperSpider.cs → ...s/WebReaper.AzureFuncs/WebReaperSpider.cs
@@ -9,12 +9,12 @@
 
 namespace WebReaper.AzureFuncs
 {
-    public class ExoScraperSpider
+    public class WebReaperSpider
     {
         public IVisitedLinkTracker LinkTracker { get; }
         public CosmosSink CosmosSink { get; }
 
-        public ExoScraperSpider(IVisitedLinkTracker linkTracker, CosmosSink cosmosSink)
+        public WebReaperSpider(IVisitedLinkTracker linkTracker, CosmosSink cosmosSink)
         {
             LinkTracker = linkTracker;
             CosmosSink = cosmosSink;

diff --git a/WebReaper/Builders/ScraperEngineBuilder.cs b/WebReaper/Builders/ScraperEngineBuilder.cs
@@ -328,9 +328,6 @@ public ScraperEngineBuilder WithParallelismDegree(int parallelismDegree)
 
     public async Task<ScraperEngine> BuildAsync()
     {
-        await _visitedLinksTracker.Initialization;
-        await Scheduler.Initialization;
-
         SpiderBuilder.WithConfigStorage(ConfigStorage);
 
         var config = ConfigBuilder.Build();

diff --git a/WebReaper/Core/ScraperEngine.cs b/WebReaper/Core/ScraperEngine.cs
@@ -37,6 +37,8 @@ public class ScraperEngine
 
     public async Task RunAsync(CancellationToken cancellationToken = default)
     {
+        await Scheduler.Initialization;
+
         Logger.LogInformation("Start {class}.{method}", nameof(ScraperEngine), nameof(RunAsync));
 
         var config = await ConfigStorage.GetConfigAsync();

diff --git a/WebReaper/Core/Spider/Concrete/Spider.cs b/WebReaper/Core/Spider/Concrete/Spider.cs
@@ -52,6 +52,8 @@ public class Spider : ISpider
 
     public async Task<List<Job>> CrawlAsync(Job job, CancellationToken cancellationToken = default)
     {
+        await LinkTracker.Initialization;
+
         var config = await ScraperConfigStorage.GetConfigAsync();
 
         if (config.UrlBlackList.Contains(job.Url)) return Enumerable.Empty<Job>().ToList();

diff --git a/WebReaper/todo.md b/WebReaper/todo.md
@@ -4,7 +4,7 @@
 - [X] Add flag for headless/not headless chrome
 - [X] Clean up on start in sink, link tracker, scheduler
 - [ ] Remove nested builders
-- [ ] ~~Use factory for async object creation~~
+- [ ] Use factory for async object creation?
 - [ ] Write tests using specflow
 - [ ] Separate nuget packages for MongoDb, Cosmos Db, Redis, etc.
 - [ ] Add support for xpath and regex