Feat/dff/voice skill (#362)

* moving files from a broken fork

* adding tg token

* changing the name of an attribute to be in line with dp-agent

* an attempt to fix []'s and a fix in rule based skill selector

* fixing attributes

* fixing validation

* small actor.py fix

* testing

* removing redundant string operations

* a minor fix that should enable .oga -> .wav

* using subprocess ffmpeg to convert audio

* adding missing files

* a facepalm fix

* another facepalm fix

* keyerror fix

* model path fix

* minor refactor

* adding support for mp3 and MP3

* dcase-2020 -> dcase-2022

* adding experiments.zip to project

* apt install gdown to download model files

* pip install gdown to download model files

* correct download links from gdrive

* gdown -> wget

* gdown -> wget 2

* fixing dockerfile

* another dockerfile fix

* adding fire to reqs

* fixing new audio-captioning

* mkdir -> makedirs

* small error fix

* debugging commit 1

* debugging commit 2

* debugging commit 3

* debugging commit 4

* debugging commit 5

* debugging commit 6

* debugging commit 7

* debugging commit 8

* debugging commit 9

* first working demo commit

* added flac and ogg formats, minor refactoring

* purging audio files dir before inference

* minor fix

* adding video notes support

* removed hardcoded tg_token

* model, service and docker files upgraded

* captioning model fixed

* image captioning restored

* agent commit changed, image_captioning temporary excluded

* Old agent

* Fixed all but pipeline_conf.json

* pipeline_conf.json fixed according to dev

* actor.py fixed according to dev

* Frommage removed

* dream_voice distributive created

* dream_voice distributive restored

* dream_multimodal updated to dev

* dream_multimodal updated to dev 2.0

* dream_multimodal updated to dev 3.0

* dream_multimodal updated to dev 4.0

* formatters fix in progress

* dp_formatters updates

* codestyle fixed, skill messages changed

* badlisted words removed from dream_voice

* pr fix

* added tests to voice skill

* voice_service fixed, tests added

* another pr fix

* pr fix

* voice formatter fixed

* another pr fix

* captions -> caption, small pr fix

* codestyle fix

* flake8 fix

* typo fixed

* codestyle fix

* tests fixed for voice skill

* another typo fixed

* reverted voice_service dockerfile

* minor pr fix

* voice service video fix

* voice service video fix 2

* git repo commit hardcoded

---------

Co-authored-by: Alexandra Dolidze <dolidze@gpu4.ipavlov.mipt.ru>

assistant_dists/dream_voice/cpu.yml

@@ -0,0 +1,14 @@
+version: '3.7'
+services:
+  dialogpt:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  intent-catcher:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""
+  sentence-ranker:
+    environment:
+      DEVICE: cpu
+      CUDA_VISIBLE_DEVICES: ""

assistant_dists/dream_voice/db_conf.json

@@ -0,0 +1,6 @@
+{
+    "host": "DB_HOST",
+    "port": "DB_PORT",
+    "name": "DB_NAME",
+    "env": true
+}

assistant_dists/dream_voice/dev.yml

@@ -0,0 +1,68 @@
+# С такими volumes удобно дебажить, не нужно пересобирать контейнер каждый раз при изменении кода
+services:
+  agent:
+    volumes:
+      - ".:/dp-agent"
+    ports:
+      - 4242:4242
+  files:
+    ports:
+      - 3000:3000
+    volumes:
+      - "~/.deeppavlov/file_server:/tmp"
+  dff-program-y-skill:
+    volumes:
+      - "./skills/dff_program_y_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8008:8008
+  sentseg:
+    volumes:
+      - "./annotators/SentSeg:/src"
+    ports:
+      - 8011:8011
+  convers-evaluation-selector:
+    volumes:
+      - "./response_selectors/convers_evaluation_based_selector:/src"
+      - "./common:/src/common"
+    ports:
+      - 8009:8009
+  dff-intent-responder-skill:
+    volumes:
+      - "./skills/dff_intent_responder_skill:/src"
+      - "./common:/src/common"
+    ports:
+      - 8012:8012
+  intent-catcher:
+    volumes:
+      - "./annotators/IntentCatcherTransformers:/src"
+      - "./common:/src/common"
+      - "~/.deeppavlov:/root/.deeppavlov"
+    ports:
+      - 8014:8014
+  dialogpt:
+    volumes:
+      - "./services/dialogpt:/src"
+      - "./common:/src/common"
+      - "~/.deeppavlov/cache:/root/.cache"
+    ports:
+      - 8125:8125
+  sentence-ranker:
+    volumes:
+      - "./services/sentence_ranker:/src"
+      - "~/.deeppavlov/cache:/root/.cache"
+    ports:
+      - 8128:8128
+  voice-service:
+    volumes:
+      - "~/.deeppavlov/file_server:/tmp"
+      - "./services/voice_service:/src"
+    ports:
+      - 8333:8333
+  dff-voice-skill:
+    volumes:
+      - "./skills/dff_voice_skill:/src"
+      - "~/.deeppavlov/file_server:/tmp"
+    ports:
+      - 8444:8444
+version: "3.7"

assistant_dists/dream_voice/docker-compose.override.yml

@@ -0,0 +1,201 @@
+services:
+  agent:
+    command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_voice/pipeline_conf.json'
+    environment:
+      WAIT_HOSTS: "dff-program-y-skill:8008, sentseg:8011, convers-evaluation-selector:8009, 
+          dff-intent-responder-skill:8012, intent-catcher:8014,
+          dialogpt:8125, sentence-ranker:8128,
+          voice-service:8333, dff-voice-skill:8444"
+      WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1200}
+      HIGH_PRIORITY_INTENTS: 1
+      RESTRICTION_FOR_SENSITIVE_CASE: 1
+      ALWAYS_TURN_ON_ALL_SKILLS: 0
+      LANGUAGE: EN
+      FALLBACK_FILE: fallbacks_dream_en.json
+
+  files:
+    image: julienmeerschart/simple-file-upload-download-server
+
+  dff-program-y-skill:
+    env_file: [.env]
+    build:
+      args:
+        SERVICE_PORT: 8008
+        SERVICE_NAME: dff_program_y_skill
+        LANGUAGE: EN
+      context: .
+      dockerfile: ./skills/dff_program_y_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8008 --reload
+    deploy:
+      resources:
+        limits:
+          memory: 1024M
+        reservations:
+          memory: 1024M
+
+
+  sentseg:
+    env_file: [.env]
+    build:
+      context: ./annotators/SentSeg/
+    command: flask run -h 0.0.0.0 -p 8011
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 1.5G
+        reservations:
+          memory: 1.5G
+
+  convers-evaluation-selector:
+    env_file: [.env]
+    build:
+      args:
+        TAG_BASED_SELECTION: 1
+        CALL_BY_NAME_PROBABILITY: 0.5
+        PROMPT_PROBA: 0.1
+        ACKNOWLEDGEMENT_PROBA: 0.3
+        PRIORITIZE_WITH_REQUIRED_ACT: 0
+        PRIORITIZE_NO_DIALOG_BREAKDOWN: 0
+        PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0
+        IGNORE_DISLIKED_SKILLS: 0
+        GREETING_FIRST: 1
+        RESTRICTION_FOR_SENSITIVE_CASE: 1
+        PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 1
+        MAX_TURNS_WITHOUT_SCRIPTS: 7
+        ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 1
+        PRIORITIZE_SCRIPTED_SKILLS: 0
+        CONFIDENCE_STRENGTH: 0.8
+        CONV_EVAL_STRENGTH: 0.4
+        PRIORITIZE_HUMAN_INITIATIVE: 1
+        QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8
+        LANGUAGE: EN
+        FALLBACK_FILE: fallbacks_dream_en.json
+      context: .
+      dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile
+    command: flask run -h 0.0.0.0 -p 8009
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+        reservations:
+          memory: 256M
+
+  dff-intent-responder-skill:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8012
+        SERVICE_NAME: dff_intent_responder_skill
+        INTENT_RESPONSE_PHRASES_FNAME: intent_response_phrases.json
+      context: .
+      dockerfile: ./skills/dff_intent_responder_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8012 --reload
+    deploy:
+      resources:
+        limits:
+          memory: 128M
+        reservations:
+          memory: 128M
+
+  intent-catcher:
+    env_file: [.env]
+    build:
+      context: .
+      dockerfile: ./annotators/IntentCatcherTransformers/Dockerfile
+      args:
+        SERVICE_PORT: 8014
+        CONFIG_NAME: intents_model_dp_config.json
+        INTENT_PHRASES_PATH: intent_phrases.json
+    command:  python -m flask run -h 0.0.0.0 -p 8014
+    environment:
+      - FLASK_APP=server
+      - CUDA_VISIBLE_DEVICES=0
+    deploy:
+      resources:
+        limits:
+          memory: 3.5G
+        reservations:
+          memory: 3.5G
+
+  dialogpt:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8125
+        SERVICE_NAME: dialogpt
+        PRETRAINED_MODEL_NAME_OR_PATH: microsoft/DialoGPT-medium
+        N_HYPOTHESES_TO_GENERATE: 5
+        CONFIG_NAME: dialogpt_en.json
+        MAX_HISTORY_DEPTH: 2
+      context: .
+      dockerfile: ./services/dialogpt/Dockerfile
+    command: flask run -h 0.0.0.0 -p 8125
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+        reservations:
+          memory: 2G
+
+  sentence-ranker:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8128
+        SERVICE_NAME: sentence_ranker
+        PRETRAINED_MODEL_NAME_OR_PATH: sentence-transformers/all-MiniLM-L6-v2
+      context: ./services/sentence_ranker/
+    command: flask run -h 0.0.0.0 -p 8128
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 3G
+        reservations:
+          memory: 3G
+
+  voice-service:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8333
+        SERVICE_NAME: voice_service
+      context: ./services/voice_service/
+    command: python -m flask run -h 0.0.0.0 -p 8333
+    environment:
+      - CUDA_VISIBLE_DEVICES=0
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 5G
+        reservations:
+          memory: 5G
+
+  dff-voice-skill:
+    env_file: [ .env ]
+    build:
+      args:
+        SERVICE_PORT: 8444
+        SERVICE_NAME: dff_voice_skill
+      context: .
+      dockerfile: ./skills/dff_voice_skill/Dockerfile
+    command: gunicorn --workers=1 server:app -b 0.0.0.0:8444 --reload
+    environment:
+      - FLASK_APP=server
+    deploy:
+      resources:
+        limits:
+          memory: 5G
+        reservations:
+          memory: 5G       
+version: '3.7'