Scrapper Added using Search API and AI Filtering

src/.dockerignore

@@ -0,0 +1,30 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+!**/.gitignore
+!.git/HEAD
+!.git/config
+!.git/packed-refs
+!.git/refs/heads/**

src/Backend/Backend.csproj

@@ -7,6 +7,11 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.AI.Agents.Persistent" Version="1.1.0" />
+    <PackageReference Include="Azure.AI.Inference" Version="1.0.0-beta.5" />
+    <PackageReference Include="Azure.AI.Projects" Version="1.0.0" />
+    <PackageReference Include="Azure.Core" Version="1.49.0" />
+    <PackageReference Include="Azure.Identity" Version="1.17.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.23.0" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.54.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.23.0" />

src/Backend/Controllers/AdminController.cs

@@ -0,0 +1,69 @@
+
+namespace Backend.Controllers
+{
+    using Backend.Operations;
+    using Common.Managers;
+    using Microsoft.AspNetCore.Mvc;
+
+    [ApiController]
+    [Route("api/admin")]
+    public class AdminController : ControllerBase
+    {
+        private readonly ILogger<AdminController> logger;
+        private readonly ScrapperRunner scrapperRunner;
+
+        public AdminController(ILogger<AdminController> logger, ScrapperRunner scrapperRunner)
+        {
+            this.logger = logger;
+            this.scrapperRunner = scrapperRunner;
+        }
+
+        [HttpGet]
+        [Route("scrappers/trigger/{scrapperId}")]
+        public ActionResult<string> TriggerScrapperRun(string scrapperId)
+        {
+            // _ = Task.Run(async () => await scrapperRunner.RunScrapperAsync(scrapperId));
+            return Ok($"[Dummy]: Scrapper run triggered for id: {scrapperId}");
+        }
+
+        [HttpPut]
+        [Route("scrappers/trigger/{scrapperId}")]
+        public ActionResult<string> EnableScrapper(string scrapperId)
+        {
+            this.scrapperRunner.EnableScrapper(scrapperId);
+            return Ok($"Scrapper enabled for id: {scrapperId}");
+        }
+
+        [HttpDelete]
+        [Route("scrappers/trigger/{scrapperId}")]
+        public ActionResult<string> DisableScrapper(string scrapperId)
+        {
+            this.scrapperRunner.DisableScrapper(scrapperId);
+            return Ok($"Scrapper disabled for id: {scrapperId}");
+        }
+
+        [HttpGet]
+        [Route("scrappers/background/start")]
+        public ActionResult<string> StartScrappersInBackground()
+        {
+            this.scrapperRunner.StartBackgroundRunner();
+            return Ok($"Background scrapper runs started. Current State: {this.scrapperRunner.CurrentState}");
+        }
+
+        [HttpGet]
+        [Route("scrappers/background/stop")]
+        public ActionResult<string> StopScrappersInBackground()
+        {
+            this.scrapperRunner.StopBackgroundRunner();
+            return Ok($"Background scrapper runs stopped. Current State: {this.scrapperRunner.CurrentState}");
+        }
+
+        [HttpGet]
+        [Route("scrappers/background/status")]
+        public ActionResult<string> GetScrappersInBackgroundStatus()
+        {
+            this.scrapperRunner.StopBackgroundRunner();
+            return Ok($"{this.scrapperRunner.GetStatus()}");
+        }
+    }
+}

src/Backend/Controllers/JobSearchController.cs

@@ -0,0 +1,60 @@
+namespace Backend.Controllers
+{
+    using Backend.Operations;
+    using Microsoft.AspNetCore.Mvc;
+    using Common.Models.Public;
+    using Common.Repositories;
+    using System.Threading.Tasks;
+    using Common.Managers;
+    using Common.Engines;
+    using Common.Queries;
+    using Common.DatabaseModels;
+
+    [ApiController]
+    [Route("api/jobs")]
+    public class JobSearchController : ControllerBase
+    {
+        private readonly JobsRepository jobsRepository;
+        private readonly ILogger<JobSearchController> logger;
+        public JobSearchController(JobsRepository jobsRepository, ILogger<JobSearchController> logger)
+        {
+            this.logger = logger;
+            this.jobsRepository = jobsRepository;
+        }
+
+        [HttpPost]
+        [Route("search")]
+        public async Task<ActionResult<List<ScrappedJob>>> SearchJobs([FromBody] JobQuery jobquery)
+        {
+            return Ok(await jobsRepository.GetJobsFromQuery(jobquery));
+        }
+
+        [HttpGet]
+        [Route("latest")]
+        public async Task<ActionResult<string>> GetLatestJobsFromDb(
+            [FromQuery] string location = "India",
+            [FromQuery] string level = "Mid")
+        {
+            return Content(JobListView.RenderScrappedJobsHtml(await this.jobsRepository.GetJobsEasyQueryAsync(location, level)), "text/html");
+        }
+
+        [HttpGet]
+        [Route("lastOneDay")]
+        public async Task<ActionResult<string>> GetLastOneDayJobsFromDb()
+        {
+            return Ok(await this.jobsRepository.GetAllJobsInLastOneDay());
+        }
+
+        [HttpGet]
+        [Route("profile/{id}")]
+        public async Task<ActionResult<string>> GetJobById(string id)
+        {
+            var job = await this.jobsRepository.GetJobByIdAsync(id);
+            if (job != null)
+            {
+                return Ok(job);
+            }
+            return Ok("Not found.");
+        }
+    }
+}

src/Backend/Controllers/ProblemsController.cs

@@ -1,7 +1,7 @@
 namespace Backend.Controllers
 {
     using Backend.Filters;
-    using Backend.Models.Public;
+    using Common.Models.Public;
     using Backend.Operations;
     using Common.Models;
     using Microsoft.AspNetCore.Mvc;

src/Backend/Controllers/ScrapperSettingsController.cs

@@ -0,0 +1,64 @@
+using Common.DatabaseModels;
+using Common.Engines;
+using Common.Managers;
+using Common.Models;
+using Common.Models.Public;
+using Common.Repositories;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Backend.Controllers
+{
+    [ApiController]
+    [Route("api/jobs/scrappers")]
+    public class ScrapperSettingsController : ControllerBase
+    {
+        private readonly JobScrapperSettingsManager _settingsManager;
+
+        private readonly ILogger<JobSearchController> _logger;
+
+        public ScrapperSettingsController(JobScrapperSettingsManager jobScrapperSettingsManager,
+            ILogger<JobSearchController> logger)
+        {
+            _settingsManager = jobScrapperSettingsManager;
+            _logger = logger;
+        }
+
+        [HttpGet]
+        [Route("")]
+        public async Task<ActionResult<List<JobScrapperSettings>>> GetAllJobScrappers()
+        {
+            // Placeholder implementation for getting all scrappers
+            return Ok(await _settingsManager.GetAllSettings());
+        }
+
+        [HttpPut]
+        [Route("{id}")]
+        public async Task<ActionResult<JobScrapperSettings>> UpdateJobScrapperSettings(string id, [FromBody] ScrapperSettings settings)
+        {
+            try
+            {
+                return Ok(await _settingsManager.CreateOrUpdateSettings(id, settings));
+            }
+            catch (InvalidOperationException ex)
+            {
+                return BadRequest(ex.Message);
+            }
+        }
+
+        [HttpPost]
+        [Route("add")]
+        public async Task<ActionResult<JobScrapperSettings>> CreateNewJobScrapperSettings([FromBody] ScrapperSettings settings)
+        {
+            return BadRequest("Use PUT api/jobs/scrappers/{id} to create or update scrapper settings.");
+            // return Ok(await _settingsManager.CreateOrUpdateSettings(string.Empty, settings));
+        }
+
+        [HttpGet]
+        [Route("{id}")]
+        public async Task<ActionResult<JobScrapperSettings>> GetJobScrapperSettings(string id)
+        {
+            // Placeholder implementation for getting scrapper settings
+            return Ok(await _settingsManager.GetSettingsById(id));
+        }
+    }
+}

src/Backend/Filters/ProblemFilter.cs

@@ -1,8 +1,9 @@
+using Common;
 using Common.Models;
 
 namespace Backend.Filters
 {
-    public class ProblemFilter : IFilter
+    public class ProblemFilter : IFilter<Problem>
     {
         private int skip = 0;
         private int limit = 50;

src/Backend/Operations/DataProvider.cs

@@ -1,6 +1,7 @@
 namespace Backend.Operations
 {
     using Backend.Filters;
+    using Common;
     using Common.Cache;
     using Common.Constants;
     using Common.Models;
@@ -15,7 +16,7 @@ public DataProvider([FromKeyedServices(CacheConstants.ProblemCacheKey)] ICache p
             _logger = logger;
         }
 
-        public async Task<List<Problem>> GetProblemsAsync(IFilter? filter = null)
+        public async Task<List<Problem>> GetProblemsAsync(IFilter<Problem>? filter = null)
         {
             var allProblems = await GetAllProblemsAsync();
             if (filter != null)