Merge pull request #187 from Joystream/main

Update `dev` from `master`

.dockerignore

@@ -0,0 +1,49 @@
+node_modules/
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# compiled output
+/dist
+/lib
+/tmp
+/out-tsc
+/local
+
+# dependencies
+**/node_modules
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+# misc
+/.sass-cache
+/connect.lock
+/coverage
+/libpeerconnection.log
+npm-debug.log
+yarn-error.log
+testem.log
+/typings
+error.log
+
+# System Files
+.DS_Store
+Thumbs.db
+**/node_modules/
+
+# Serverless
+.serverless
+.webpack
+.env

CHANGELOG.md

@@ -1,3 +1,12 @@
+### 1.4.0
+
+- Adds support for whitelisting a channel such that a whitelisted channel will be exempted from requirements check when signing up for the YPP program. Adds `POST /channels/whitelist` endpoint to whitelist a channel/s & `DELETE /channels/whitelist/{ytChannelHandle}` endpoint to remove a channel from whitelist.
+
+### 1.3.0
+
+- Integrates ElasticSearch alerting feature based on the filtration criteria set on the ingested logs.
+- Adds `./scripts/create-elasticsearch-alert.sh` script to automate the creation of _Kibana Alert Rules_ and _Action Connectors_ for Alerting purpose.
+
 ### 1.2.0
 
 - Changed the order by which videos should be synced, now the videos are synced in the order of their `publishedAt` date instead of `views`. This is done to ensure that the videos are synced in the order they were published on Youtube.

Dockerfile

@@ -0,0 +1,16 @@
+FROM node:14
+
+# Set the working directory to /youtube-synch
+WORKDIR /youtube-synch
+
+# Copy the contents of the current directory to /youtube-synch
+COPY . /youtube-synch
+
+# Install dependencies
+RUN yarn install
+
+# Build the project
+RUN yarn build
+
+# Set the command to run when a container based on the image is started
+CMD ["./bin/run", "start"]

README.md

@@ -86,3 +86,30 @@ For running Youtube-synch service as a docker container, run `docker-compose up
 There is a CLI command for doing unauthorized replication/syncing of Youtube Channel's videos on joystream. For more information see [sync:addUnauthorizedChannelForSyncing](./src/cli/docs/sync.md)
 
 Also, if you want to sync multiple unauthorized channels, you can use [sync:syncMultipleUnauthorizedChannels](./src/cli/docs/sync.md) command
+
+# Elasticsearch Alerting & Monitoring
+
+The YT-Synch service logs can be sent to Elasticsearch instance, which then can be used to create alerting & monitoring rules based on the defined criteria. There is a [script](scripts/create-elasticsearch-alert.sh) designed to automate the creation of _Kibana Alert Rules_ and _Action Connectors_ for monitoring the Youtube Synchronization Service. The script creates an alert rule that queries Elasticsearch for any errors occurring in the service within the past some time (in minutes). If any errors are found, the script triggers email and Discord notifications to inform recipients of the issue. The script also creates the necessary connectors for sending these notifications.
+
+## Environment Variables
+
+The script uses the following environment variables to configure the alert rule and connectors:
+
+- `KIBANA_URL`: The URL of the Kibana instance (default: `http://localhost:5601`).
+- `ELASTIC_USERNAME`: The username for accessing Kibana (default: `elastic`).
+- `ELASTIC_PASSWORD`: The password for accessing Kibana.
+- `EMAIL_RECIPIENTS`: A comma-separated list of email addresses to receive email notifications.
+- `DISCORD_WEBHOOK_URL`: The webhook URL for sending notifications to Discord.
+- `THRESHOLD`: The threshold for triggering the alert (default: `10`).
+- `EMAIL_CONNECTOR_NAME`: The name of the email connector to be created in Kibana (default: `Elastic-Cloud-SMTP`).
+- `WEBHOOK_CONNECTOR_NAME`: The name of the webhook connector to be created in Kibana (default: `Discord Webhook`).
+- `ALERT_RULE_NAME`: The name of the alert rule to be created in Kibana (default: `YT-Sync-Service-Alert`).
+
+## How to Run the Script
+
+1. Ensure that you have the required tools installed on your system: `bash`, `curl`, and `jq`.
+2. Set the necessary environment variables in your shell or export them in a `.env` file.
+3. Make the script executable by running `chmod +x scripts/create-elasticsearch-alert.sh`.
+4. Run the script with `./scripts/create-elasticsearch-alert.sh`.
+
+Upon successful execution, the script will create an alert rule and the required connectors in Kibana. If any errors are encountered, the script will display an error message with details on the issue.

config.yml

@@ -14,6 +14,9 @@ logs:
   # elastic:
   #   level: info
   #   endpoint: http://localhost:9200
+  #   auth:
+  #     username: elastic-username
+  #     password: elastic-password
 intervals: # in minutes
   youtubePolling: 30
   contentProcessing: 1
@@ -50,12 +53,13 @@ joystream:
       - mnemonic: 'escape naive annual throw tragic achieve grunt verify cram note harvest problem'
       # - suri: //Alice
 creatorOnboardingRequirements:
-  minimumSubscribersCount: 1
-  minimumVideoCount: 2
-  minimumVideoAgeHours: 1
-  minimumChannelAgeHours: 1
   # ! Production values
-  # minimumSubscribersCount: 50
-  # minimumVideoCount: 10
-  # minimumVideoAgeHours: 720 # (24 hours x 30 days) 1 month
-  # minimumChannelAgeHours: 2160 # (3 x 24 hours x 30 days) 3 months
+  minimumSubscribersCount: 50
+  minimumVideoCount: 10
+  minimumVideoAgeHours: 720 # (24 hours x 30 days) 1 month
+  minimumChannelAgeHours: 2160 # (3 x 24 hours x 30 days) 3 months
+  # ! Development values
+  # minimumSubscribersCount: 1
+  # minimumVideoCount: 2
+  # minimumVideoAgeHours: 1
+  # minimumChannelAgeHours: 1

docker-compose.elasticsearch.yml

@@ -0,0 +1,46 @@
+version: '3.4'
+
+services:
+  # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/8.7/docker.html
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.7.0
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
+      - xpack.security.enabled=true
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-password}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es-data:/usr/share/elasticsearch/data
+    ports:
+      - 127.0.0.1:9200:9200
+  # Ref: https://www.elastic.co/guide/en/kibana/8.7/docker.html
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.7.0
+    container_name: kibana
+    ports:
+      - 127.0.0.1:5601:5601
+    depends_on:
+      - elasticsearch
+    environment:
+      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
+      ELASTICSEARCH_SERVICEACCOUNTTOKEN: ${ELASTICSEARCH_SERVICEACCOUNTTOKEN}
+      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY}
+
+volumes:
+  es-data:
+    driver: local
+  cache:
+    driver: local
+  data:
+    driver: local
+
+networks:
+  joystream:
+    external: true
+    name: youtube-synch_default

docker-compose.yml

@@ -1,24 +1,23 @@
 version: '3.4'
 services:
-  youtube-synch-service:
-    image: node:14
-    container_name: youtube-synch-service
+  youtube-synch:
+    image: youtube-synch:${IMAGE_TAG}
+    container_name: youtube-synch
     restart: on-failure
-    network_mode: "host"
+    build:
+      context: .
+      dockerfile: Dockerfile
     volumes:
-      - youtube-synch-logs:/logs
-      - youtube-synch-data:/data
+      - logs:/logs
+      - data:/data
       # mount the AWS credentials file to docker container home directory
       - ~/.aws/:/root/.aws:ro
-      - type: bind
-        source: .
-        target: /youtube-synch
-    working_dir: /youtube-synch
     env_file:
       # relative to working directory where docker-compose was run from
       - .env
     environment:
       DEPLOYMENT_ENV: ${DEPLOYMENT_ENV}
+    #   YT_SYNCH__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
     #   YT_SYNCH__ENDPOINTS__QUERY_NODE: ${YT_SYNCH__ENDPOINTS__QUERY_NODE}
     #   YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS: ${YT_SYNCH__ENDPOINTS__JOYSTREAM_NODE_WS}
     #   YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT}
@@ -32,10 +31,11 @@ services:
     #   YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID: ${YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID}
     #   YT_SYNCH__JOYSTREAM__APP__NAME: ${YT_SYNCH__JOYSTREAM__APP__NAME}
     #   YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED: ${YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED}
-    command: [ './bin/run', 'start' ]
+    ports:
+      - 127.0.0.1:${YT_SYNCH__HTTP_API__PORT}:${YT_SYNCH__HTTP_API__PORT}
 
 volumes:
-  youtube-synch-logs:
+  logs:
     driver: local
-  youtube-synch-data:
+  data:
     driver: local

package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-sync",
-  "version": "1.2.1",
+  "version": "1.4.0",
   "license": "MIT",
   "scripts": {
     "postpack": "rm -f oclif.manifest.json",
@@ -9,8 +9,8 @@
     "start": "DEPLOYMENT_ENV=dev ./bin/run start",
     "lint": "eslint ./src --ext .ts",
     "start:prod": "./bin/run start",
-    "docker:start": "DEPLOYMENT_ENV=dev docker-compose up -d",
-    "docker:start:prod": "docker-compose up -d",
+    "docker:start": "DEPLOYMENT_ENV=dev IMAGE_TAG=$(./scripts/code-shasum.sh) docker-compose up -d",
+    "docker:start:prod": "IMAGE_TAG=$(./scripts/code-shasum.sh) docker-compose up -d",
     "dynamodb:start": "yarn dynamodb:stop && export DEPLOYMENT_ENV=local && ./src/infrastructure/deploy.sh",
     "dynamodb:stop": "export DEPLOYMENT_ENV=local REMOVE_PULUMI_STACK=true && ./src/infrastructure/destroy.sh",
     "generate:docs:cli": "yarn oclif-dev readme --multi --dir ./src/cli/docs",

scripts/code-shasum.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Compute a hash over files related to building youtube-synch docker image
+
+# Assuming cargo workspace root is same as the git repo root
+cd $(git rev-parse --show-toplevel)
+
+TAR=tar
+SED=sed
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    TAR=gtar
+    SED=gsed
+fi
+
+# sort/owner/group/mtime arguments only work with gnu version of tar!
+${TAR} -c --sort=name --owner=root:0 --group=root:0 --mode 644 --mtime='UTC 2020-01-01' \
+    src \
+    Dockerfile \
+    config.yml \
+    package.json \
+    bin |
+    if [[ -n "$RUNTIME_PROFILE" ]]; then ${SED} '$a'"$RUNTIME_PROFILE"; else tee; fi |
+    shasum |
+    cut -d " " -f 1