Merge pull request #46 from purple-emily/change-to-single-env-file

Simplify the environment variables

.env.example

@@ -0,0 +1,39 @@
+# General environment variables
+TZ=London/Europe
+
+# PostgreSQL
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=knightcrawler
+
+# MongoDB
+MONGODB_HOST=mongodb
+MONGODB_PORT=27017
+MONGODB_DB=knightcrawler
+MONGO_INITDB_ROOT_USERNAME=mongo
+MONGO_INITDB_ROOT_PASSWORD=mongo
+
+# Addon
+DEBUG_MODE=false
+
+# Consumer
+RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30
+QUEUE_NAME=ingested
+JOB_CONCURRENCY=5
+JOBS_ENABLED=true
+MAX_SINGLE_TORRENT_CONNECTIONS=10
+TORRENT_TIMEOUT=30000
+UDP_TRACKERS_ENABLED=true
+
+# Producer
+RabbitMqConfiguration__Host=rabbitmq
+RabbitMqConfiguration__QueueName=ingested
+RabbitMqConfiguration__Username=guest
+RabbitMqConfiguration__Password=guest
+RabbitMqConfiguration__Durable=true
+RabbitMqConfiguration__MaxQueueSize=0
+RabbitMqConfiguration__MaxPublishBatchSize=500
+RabbitMqConfiguration__PublishIntervalInSeconds=10
+GithubSettings__PAT=

.gitignore

@@ -2,6 +2,7 @@
 .now
 .DS_Store
 .idea
+.env
 
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.

README.md

@@ -13,6 +13,7 @@ A self-hosted Stremio addon for streaming torrents via a debrid service.
   - [Overview](#overview)
   - [Using](#using)
     - [Initial setup (optional)](#initial-setup-optional)
+    - [Environment Setup](#environment-setup)
     - [Run the project](#run-the-project)
     - [Monitoring with Grafana and Prometheus (Optional)](#monitoring-with-grafana-and-prometheus-optional)
       - [Accessing RabbitMQ Management](#accessing-rabbitmq-management)
@@ -60,11 +61,22 @@ We can search DebridMediaManager hash lists which are hosted on GitHub. This all
    GithubSettings__PAT=<YOUR TOKEN HERE>
    ```
 
+
+### Environment Setup
+
+Before running the project, you need to set up the environment variables. Copy the `.env.example` file to `.env`:
+
+```sh
+cp .env.example .env
+```
+
+Then set any of th values you'd like to customize.
+
 ### Run the project
 
 Open a terminal in the directory and run the command:
 
-``` sh
+```sh
 docker compose up -d
 ```
 
@@ -108,6 +120,7 @@ Now, you can use these dashboards to monitor RabbitMQ and Postgres metrics.
 
 Note: If you encounter issues with missing or unavailable data in Grafana, please ensure on [Prometheus's target page](http://127.0.0.1:9090/targets) that the RabbitMQ target is up and running.
 
+
 ## Importing external dumps
 
 A brief record of the steps required to import external data, in this case the rarbg dump which can be found on RD:

docker-compose.yaml

@@ -35,10 +35,9 @@ x-apps: &knightcrawler-app
 services:
   postgres:
     image: postgres:latest
+    env_file:
+      - .env
     environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: knightcrawler
       PGUSER: postgres # needed for healthcheck.
     ports:
       - "5432:5432"
@@ -51,9 +50,8 @@ services:
 
   mongodb:
     image: mongo:latest
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: mongo
-      MONGO_INITDB_ROOT_PASSWORD: mongo
+    env_file:
+      - .env
     ports:
       - "27017:27017"
     volumes:
@@ -81,7 +79,7 @@ services:
       context: src/producer
       dockerfile: Dockerfile
     env_file:
-      - env/producer.env
+      - .env
     <<: *knightcrawler-app
     networks:
       - knightcrawler-network
@@ -91,7 +89,7 @@ services:
       context: src/node/consumer
       dockerfile: Dockerfile
     env_file:
-      - env/consumer.env
+      - .env
     deploy:
       replicas: 3
     <<: *knightcrawler-app
@@ -105,7 +103,7 @@ services:
     ports:
       - "7000:7000"
     env_file:
-      - env/addon.env
+      - .env
     <<: *knightcrawler-app
     networks:
       - knightcrawler-network

src/node/addon/src/lib/cache.js

@@ -1,6 +1,7 @@
 import cacheManager from 'cache-manager';
 import mangodbStore from 'cache-manager-mongodb';
-import { isStaticUrl }  from '../moch/static.js';
+import { cacheConfig } from './config.js';
+import { isStaticUrl } from '../moch/static.js';
 
 const GLOBAL_KEY_PREFIX = 'knightcrawler-addon';
 const STREAM_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|stream`;
@@ -14,90 +15,87 @@ const AVAILABILITY_EMPTY_TTL = 30 * 60; // 30 minutes
 const MESSAGE_VIDEO_URL_TTL = 60; // 1 minutes
 // When the streams are empty we want to cache it for less time in case of timeouts or failures
 
-const MONGO_URI = process.env.MONGODB_URI;
-const NO_CACHE = process.env.NO_CACHE || false;
-
 const memoryCache = initiateMemoryCache();
 const remoteCache = initiateRemoteCache();
 
 function initiateRemoteCache() {
-  if (NO_CACHE) {
-    return null;
-  } else if (MONGO_URI) {
-    return cacheManager.caching({
-      store: mangodbStore,
-      uri: MONGO_URI,
-      options: {
-        collection: 'knightcrawler_addon_collection',
-        socketTimeoutMS: 120000,
-        useNewUrlParser: true,
-        useUnifiedTopology: false,
-        ttl: STREAM_EMPTY_TTL
-      },
-      ttl: STREAM_EMPTY_TTL,
-      ignoreCacheErrors: true
-    });
-  } else {
-    return cacheManager.caching({
-      store: 'memory',
-      ttl: STREAM_EMPTY_TTL
-    });
-  }
+    if (cacheConfig.NO_CACHE) {
+        return null;
+    } else if (cacheConfig.MONGO_URI) {
+        return cacheManager.caching({
+            store: mangodbStore,
+            uri: cacheConfig.MONGO_URI,
+            options: {
+                collection: 'knightcrawler_addon_collection',
+                socketTimeoutMS: 120000,
+                useNewUrlParser: true,
+                useUnifiedTopology: false,
+                ttl: STREAM_EMPTY_TTL
+            },
+            ttl: STREAM_EMPTY_TTL,
+            ignoreCacheErrors: true
+        });
+    } else {
+        return cacheManager.caching({
+            store: 'memory',
+            ttl: STREAM_EMPTY_TTL
+        });
+    }
 }
 
 function initiateMemoryCache() {
-  return cacheManager.caching({
-    store: 'memory',
-    ttl: MESSAGE_VIDEO_URL_TTL,
-    max: Infinity // infinite LRU cache size
-  });
+    return cacheManager.caching({
+        store: 'memory',
+        ttl: MESSAGE_VIDEO_URL_TTL,
+        max: Infinity // infinite LRU cache size
+    });
 }
 
 function cacheWrap(cache, key, method, options) {
-  if (NO_CACHE || !cache) {
-    return method();
-  }
-  return cache.wrap(key, method, options);
+    if (cacheConfig.NO_CACHE || !cache) {
+        return method();
+    }
+    return cache.wrap(key, method, options);
 }
 
 export function cacheWrapStream(id, method) {
-  return cacheWrap(remoteCache, `${STREAM_KEY_PREFIX}:${id}`, method, {
-    ttl: (streams) => streams.length ? STREAM_TTL : STREAM_EMPTY_TTL
-  });
+    return cacheWrap(remoteCache, `${STREAM_KEY_PREFIX}:${id}`, method, {
+        ttl: (streams) => streams.length ? STREAM_TTL : STREAM_EMPTY_TTL
+    });
 }
 
 export function cacheWrapResolvedUrl(id, method) {
-  return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, {
-    ttl: (url) => isStaticUrl(url) ? MESSAGE_VIDEO_URL_TTL : STREAM_TTL
-  });
+    return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, {
+        ttl: (url) => isStaticUrl(url) ? MESSAGE_VIDEO_URL_TTL : STREAM_TTL
+    });
 }
 
 export function cacheAvailabilityResults(results) {
-  Object.keys(results)
-      .forEach(infoHash => {
-        const key = `${AVAILABILITY_KEY_PREFIX}:${infoHash}`;
-        const value = results[infoHash];
-        const ttl = value?.length ? AVAILABILITY_TTL : AVAILABILITY_EMPTY_TTL;
-        memoryCache.set(key, value, { ttl })
-      });
-  return results;
+    Object.keys(results)
+        .forEach(infoHash => {
+            const key = `${AVAILABILITY_KEY_PREFIX}:${infoHash}`;
+            const value = results[infoHash];
+            const ttl = value?.length ? AVAILABILITY_TTL : AVAILABILITY_EMPTY_TTL;
+            memoryCache.set(key, value, { ttl })
+        });
+    return results;
 }
 
 export function getCachedAvailabilityResults(infoHashes) {
-  const keys = infoHashes.map(infoHash => `${AVAILABILITY_KEY_PREFIX}:${infoHash}`)
-  return new Promise(resolve => {
-    memoryCache.mget(...keys, (error, result) => {
-      if (error) {
-        console.log('Failed retrieve availability cache', error)
-        return resolve({});
-      }
-      const availabilityResults = {};
-      infoHashes.forEach((infoHash, index) => {
-        if (result[index]) {
-          availabilityResults[infoHash] = result[index];
-        }
-      });
-      resolve(availabilityResults);
-    })
-  });
+    const keys = infoHashes.map(infoHash => `${AVAILABILITY_KEY_PREFIX}:${infoHash}`)
+    return new Promise(resolve => {
+        memoryCache.mget(...keys, (error, result) => {
+            if (error) {
+                console.log('Failed retrieve availability cache', error)
+                return resolve({});
+            }
+            const availabilityResults = {};
+            infoHashes.forEach((infoHash, index) => {
+                if (result[index]) {
+                    availabilityResults[infoHash] = result[index];
+                }
+            });
+            resolve(availabilityResults);
+        })
+    });
 }

src/node/addon/src/lib/config.js

@@ -0,0 +1,38 @@
+export const cacheConfig = {
+    MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
+    MONGODB_PORT: process.env.MONGODB_PORT || '27017',
+    MONGODB_DB: process.env.MONGODB_DB || 'selfhostio',
+    MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
+    MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
+    COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'selfhostio_consumer_collection',
+    NO_CACHE: parseBool(process.env.NO_CACHE, false),
+}
+
+// Combine the environment variables into a connection string
+// The combined string will look something like:
+// 'mongodb://mongo:mongo@localhost:27017/selfhostio?authSource=admin'
+cacheConfig.MONGO_URI = 'mongodb://' + cacheConfig.MONGO_INITDB_ROOT_USERNAME + ':' + cacheConfig.MONGO_INITDB_ROOT_PASSWORD + '@' + cacheConfig.MONGODB_HOST + ':' + cacheConfig.MONGODB_PORT + '/' + cacheConfig.MONGODB_DB + '?authSource=admin';
+
+export const databaseConfig = {
+    POSTGRES_HOST: process.env.POSTGRES_HOST || 'postgres',
+    POSTGRES_PORT: process.env.POSTGRES_PORT || '5432',
+    POSTGRES_DATABASE: process.env.POSTGRES_DATABASE || 'selfhostio',
+    POSTGRES_USERNAME: process.env.POSTGRES_USERNAME || 'postgres',
+    POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD || 'postgres',
+}
+
+// Combine the environment variables into a connection string
+// The combined string will look something like:
+// 'postgres://postgres:postgres@localhost:5432/selfhostio'
+databaseConfig.POSTGRES_URI = 'postgres://' + databaseConfig.POSTGRES_USERNAME + ':' + databaseConfig.POSTGRES_PASSWORD + '@' + databaseConfig.POSTGRES_HOST + ':' + databaseConfig.POSTGRES_PORT + '/' + databaseConfig.POSTGRES_DATABASE;
+
+
+function parseBool(boolString, defaultValue) {
+    const isString = typeof boolString === 'string' || boolString instanceof String;
+
+    if (!isString) {
+        return defaultValue;
+    }
+
+    return boolString.toLowerCase() === 'true' ? true : defaultValue;
+}