Change postgres configuration in the producer to use the env vars fro…

…m the stack

.env

@@ -28,7 +28,6 @@ TORRENT_TIMEOUT=30000
 UDP_TRACKERS_ENABLED=true
 
 # Producer
-ScrapeConfiguration__StorageConnectionString=host=postgres;username=postgres;password=postgres;database=knightcrawler;
 RabbitMqConfiguration__Host=rabbitmq
 RabbitMqConfiguration__QueueName=ingested
 RabbitMqConfiguration__Username=guest
@@ -37,4 +36,4 @@ RabbitMqConfiguration__Durable=true
 RabbitMqConfiguration__MaxQueueSize=0
 RabbitMqConfiguration__MaxPublishBatchSize=500
 RabbitMqConfiguration__PublishIntervalInSeconds=10
-GithubSettings__PAT=
+GithubSettings__PAT=

src/producer/Configuration/scrapers.json

@@ -1,6 +1,5 @@
 {
   "ScrapeConfiguration": {
-    "StorageConnectionString": "",
     "Scrapers": [
       {
         "Name": "SyncEzTvJob",

src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs

@@ -1,3 +1,5 @@
+using Producer.Models.Configuration;
+
 namespace Producer.Crawlers.Sites;
 
 public partial class DebridMediaManagerCrawler(

src/producer/Extensions/ConfigurationExtensions.cs

@@ -1,3 +1,5 @@
+using Producer.Models.Configuration;
+
 namespace Producer.Extensions;
 
 public static class ConfigurationExtensions

src/producer/Extensions/ServiceCollectionExtensions.cs

@@ -1,3 +1,5 @@
+using Producer.Models.Configuration;
+
 namespace Producer.Extensions;
 
 public static class ServiceCollectionExtensions
@@ -20,6 +22,7 @@ internal static IServiceCollection AddCrawlers(this IServiceCollection services)
 
     internal static IServiceCollection AddDataStorage(this IServiceCollection services)
     {
+        services.LoadConfigurationFromEnv<PostgresConfiguration>();
         services.AddTransient<IDataStorage, DapperDataStorage>();
         services.AddTransient<IMessagePublisher, TorrentPublisher>();
         return services;
@@ -36,9 +39,9 @@ internal static IServiceCollection RegisterMassTransit(this IServiceCollection s
         services.AddMassTransit(busConfigurator =>
         {
             busConfigurator.SetKebabCaseEndpointNameFormatter();
-            busConfigurator.UsingRabbitMq((context, busFactoryConfigurator) =>
+            busConfigurator.UsingRabbitMq((_, busFactoryConfigurator) =>
             {
-                busFactoryConfigurator.Host(rabbitConfig!.Host, hostConfigurator =>
+                busFactoryConfigurator.Host(rabbitConfig.Host, hostConfigurator =>
                 {
                     hostConfigurator.Username(rabbitConfig.Username);
                     hostConfigurator.Password(rabbitConfig.Password);
@@ -51,9 +54,9 @@ internal static IServiceCollection RegisterMassTransit(this IServiceCollection s
 
     internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
     {
-        var scrapeConfiguration = LoadScrapeConfiguration(services, configuration);
-        var githubConfiguration = LoadGithubConfiguration(services, configuration);
-        var rabbitConfig = LoadRabbitMQConfiguration(services, configuration);
+        var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
+        var githubConfiguration = services.LoadConfigurationFromConfig<GithubConfiguration>(configuration, GithubConfiguration.SectionName);
+        var rabbitConfig = services.LoadConfigurationFromConfig<RabbitMqConfiguration>(configuration, RabbitMqConfiguration.SectionName);
 
         services
             .AddTransient<SyncEzTvJob>()
@@ -92,46 +95,29 @@ internal static IServiceCollection AddQuartz(this IServiceCollection services, I
 
         return services;
     }
-
-    private static GithubConfiguration LoadGithubConfiguration(IServiceCollection services, IConfiguration configuration)
+
+    private static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
+        where TConfiguration : class
     {
-        var githubConfiguration = configuration.GetSection(GithubConfiguration.SectionName).Get<GithubConfiguration>();
-
-        ArgumentNullException.ThrowIfNull(githubConfiguration, nameof(githubConfiguration));
+        var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
 
-        services.TryAddSingleton(githubConfiguration);
-
-        return githubConfiguration;
-    }
-
-    private static RabbitMqConfiguration LoadRabbitMQConfiguration(IServiceCollection services, IConfiguration configuration)
-    {
-        var rabbitConfiguration = configuration.GetSection(RabbitMqConfiguration.SectionName).Get<RabbitMqConfiguration>();
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
 
-        ArgumentNullException.ThrowIfNull(rabbitConfiguration, nameof(rabbitConfiguration));
+        services.TryAddSingleton(instance);
 
-        if (rabbitConfiguration.MaxQueueSize > 0)
-        {
-            if (rabbitConfiguration.MaxPublishBatchSize > rabbitConfiguration.MaxQueueSize)
-            {
-                throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration");
-            }
-        }
-
-        services.TryAddSingleton(rabbitConfiguration);
-
-        return rabbitConfiguration;
+        return instance;
     }
-
-    private static ScrapeConfiguration LoadScrapeConfiguration(IServiceCollection services, IConfiguration configuration)
+
+    private static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
+        where TConfiguration : class
     {
-        var scrapeConfiguration = configuration.GetSection(ScrapeConfiguration.SectionName).Get<ScrapeConfiguration>();
+        var instance = Activator.CreateInstance<TConfiguration>();
 
-        ArgumentNullException.ThrowIfNull(scrapeConfiguration, nameof(scrapeConfiguration));
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
 
-        services.TryAddSingleton(scrapeConfiguration);
+        services.TryAddSingleton(instance);
 
-        return scrapeConfiguration;
+        return instance;
     }
 
     private static void AddJobWithTrigger<TJobType>(

src/producer/GlobalUsings.cs

@@ -2,6 +2,7 @@
 
 global using System.Text;
 global using System.Text.Json;
+global using System.Text.Json.Serialization;
 global using System.Text.RegularExpressions;
 global using System.Xml.Linq;
 global using Dapper;

src/producer/Models/GithubConfiguration.cs → src/producer/Models/Configuration/GithubConfiguration.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Models.Configuration;
 
 public class GithubConfiguration
 {

src/producer/Models/Configuration/PostgresConfiguration.cs

@@ -0,0 +1,29 @@
+namespace Producer.Models.Configuration;
+
+public class PostgresConfiguration
+{
+    private const string Prefix = "POSTGRES";
+    private const string HostVariable = "HOST";
+    private const string UsernameVariable = "USER";
+    private const string PasswordVariable = "PASSWORD";
+    private const string DatabaseVariable = "DB";
+    private const string PortVariable = "PORT";
+
+    private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ??
+                                         throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set");
+
+    private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ??
+                                             throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set");
+
+    private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ??
+                                             throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set");
+
+    private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ??
+                                             throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set");
+
+    private int PORT { get; init; } = int.Parse(
+        Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ??
+        throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set"));
+
+    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
+}

src/producer/Models/Configuration/RabbitMqConfiguration.cs

@@ -0,0 +1,39 @@
+namespace Producer.Models.Configuration;
+
+public class RabbitMqConfiguration
+{
+    public const string SectionName = "RabbitMqConfiguration";
+    public const string Filename = "rabbitmq.json";
+
+    public string? Host { get; set; }
+    public string? Username { get; set; }
+    public string? Password { get; set; }
+    public string? QueueName { get; set; }
+    public bool Durable { get; set; }
+    public int MaxQueueSize { get; set; }
+    public int MaxPublishBatchSize { get; set; } = 500;
+    public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
+
+    public void Validate()
+    {
+        if (MaxQueueSize == 0)
+        {
+            return;
+        }
+
+        if (MaxQueueSize < 0)
+        {
+            throw new InvalidOperationException("MaxQueueSize cannot be less than 0 in RabbitMqConfiguration");
+        }
+
+        if (MaxPublishBatchSize < 0)
+        {
+            throw new InvalidOperationException("MaxPublishBatchSize cannot be less than 0 in RabbitMqConfiguration");
+        }
+
+        if (MaxPublishBatchSize > MaxQueueSize)
+        {
+            throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration");
+        }
+    }
+}

src/producer/Models/ScrapeConfiguration.cs → src/producer/Models/Configuration/ScrapeConfiguration.cs

@@ -1,10 +1,9 @@
-namespace Producer.Models;
+namespace Producer.Models.Configuration;
 
 public class ScrapeConfiguration
 {
     public const string SectionName = "ScrapeConfiguration";
     public const string Filename = "scrapers.json";
 
     public List<Scraper> Scrapers { get; set; } = [];
-    public string StorageConnectionString { get; set; } = "";
 }

src/producer/Producer.csproj

@@ -40,6 +40,7 @@
       <None Include="Configuration\github.json">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
+      <Content Remove="Configuration\postgres.json" />
     </ItemGroup>
 
 </Project>

src/producer/Services/DapperDataStorage.cs

@@ -1,6 +1,8 @@
+using Producer.Models.Configuration;
+
 namespace Producer.Services;
 
-public class DapperDataStorage(ScrapeConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
+public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
 {
     private const string InsertTorrentSql =
         """

src/producer/Services/TorrentPublisher.cs

@@ -1,4 +1,6 @@
-namespace Producer.Services;
+using Producer.Models.Configuration;
+
+namespace Producer.Services;
 
 public class TorrentPublisher(
     ISendEndpointProvider sendEndpointProvider,