diff --git a/.github/workflows/pod_encoding.yml b/.github/workflows/pod_encoding.yml new file mode 100644 index 0000000000..32899dc835 --- /dev/null +++ b/.github/workflows/pod_encoding.yml @@ -0,0 +1,87 @@ +name: Pod Encoding Full Docker +run-name: ${{ github.actor }} is testing Pod encoding 🚀 + +on: + push: + branches: + - main + - master + - develop + - features/** + - dependabot/** + pull_request: + branches: + - main + - master + - develop +env: + DJANGO_SUPERUSER_USERNAME: "admin" + DJANGO_SUPERUSER_PASSWORD: "passwd" + DJANGO_SUPERUSER_EMAIL: "noreplay@uni.fr" + ELASTICSEARCH_TAG: "elasticsearch:7.17.18" + ELASTICSEARCH_VERION: "elasticsearch:7.17.18" + NODE_TAG: "node:19" + PYTHON_TAG: "python:3.9-buster" + REDIS_TAG: "redis:alpine3.16" + DOCKER_ENV: "full" + GECKODRIVER_VER: "v0.29.0" + FIREFOX_VER: "87.0" + +jobs: + Pod-Docker-Encoding-Actions: + runs-on: ubuntu-latest + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v4 + - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." + - run: echo "🖥️ The workflow is now ready to test your code on the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - run: echo "🍏 This job's status is ${{ job.status }}." + - name: Create settings local file + run: | + mv pod/custom/settings_local_docker_full_test.py pod/custom/settings_local.py + - name: cat settings local + run: cat pod/custom/settings_local.py + - name: Create env file containers + run: | + touch .env.dev + echo DJANGO_SUPERUSER_USERNAME=$DJANGO_SUPERUSER_USERNAME >> .env.dev + echo DJANGO_SUPERUSER_PASSWORD=$DJANGO_SUPERUSER_PASSWORD >> .env.dev + echo DJANGO_SUPERUSER_EMAIL=$DJANGO_SUPERUSER_EMAIL >> .env.dev + echo ELASTICSEARCH_TAG=$ELASTICSEARCH_TAG >> .env.dev + echo ELASTICSEARCH_VERSION=$ELASTICSEARCH_TAG >> .env.dev + echo NODE_TAG=$NODE_TAG >> .env.dev + echo PYTHON_TAG=$PYTHON_TAG >> .env.dev + echo REDIS_TAG=$REDIS_TAG >> .env.dev + echo DOCKER_ENV=full >> .env.dev + echo GECKODRIVER_VER=v0.29.0 + echo FIREFOX_VER=87.0 + - name: cat env + run: cat .env.dev + - name: make Build container + run: | + sudo rm -rf ./pod/log + sudo rm -rf ./pod/static + sudo rm -rf ./pod/node_modules + docker-compose -f ./docker-compose-full-dev-with-volumes.yml -p esup-pod build --build-arg ELASTICSEARCH_VERSION=$ELASTICSEARCH_TAG --build-arg NODE_VERSION=$NODE_TAG --build-arg PYTHON_VERSION=$PYTHON_TAG --no-cache + docker-compose -f ./docker-compose-full-dev-with-volumes.yml up --detach --force-recreate --always-recreate-deps + - name: Sleep for 60 seconds to wait run server on pod back + uses: jakejarvis/wait-action@master + with: + time: '60s' + - name: show running container + run: docker ps + - run: | + echo "🍏 Docker is UP ${{ job.status }}." + docker exec pod-back-with-volumes ps auxf + - name: run test in docker + run: docker exec pod-back-with-volumes coverage run --source='.' manage.py test_encode_transcript + - name: Stop containers + if: always() + run: docker-compose -f ./docker-compose-full-dev-with-volumes.yml down + - run: echo "END" diff --git a/dockerfile-dev-with-volumes/README.adoc b/dockerfile-dev-with-volumes/README.adoc index a902038a23..8097bcc2ae 100755 --- a/dockerfile-dev-with-volumes/README.adoc +++ b/dockerfile-dev-with-volumes/README.adoc @@ -102,7 +102,7 @@ SESSION_REDIS = { MIGRATION_MODULES = {'flatpages': 'pod.db_migrations'} # Si DOCKER_ENV = full il faut activer l'encodage et la transcription distante -# USE_DISTANT_ENCODING_TRANSCODING = True +# USE_REMOTE_ENCODING_TRANSCODING = True # ENCODING_TRANSCODING_CELERY_BROKER_URL = "redis://redis:6379/7" # pour avoir le maximum de log sur la console diff --git a/dockerfile-dev-with-volumes/pod-back/Dockerfile b/dockerfile-dev-with-volumes/pod-back/Dockerfile index 6f509e568c..e4733869fd 100755 --- a/dockerfile-dev-with-volumes/pod-back/Dockerfile +++ b/dockerfile-dev-with-volumes/pod-back/Dockerfile @@ -32,7 +32,7 @@ RUN mkdir /tmp/node_modules/ COPY --from=source-build-js /tmp/pod/node_modules/ /tmp/node_modules/ # TODO remove ES version - move it into env var RUN pip3 install --no-cache-dir -r requirements-conteneur.txt \ - && pip3 install elasticsearch==8.9.0 + && pip3 install elasticsearch==7.17.7 # ENTRYPOINT : COPY ./dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh /tmp/my-entrypoint-back.sh diff --git a/dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh b/dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh index 5e785f880c..67d58b7229 100644 --- a/dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh +++ b/dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh @@ -25,7 +25,5 @@ fi # Le serveur de développement permet de tester vos futures modifications facilement. # N'hésitez pas à lancer le serveur de développement pour vérifier vos modifications au fur et à mesure. # À ce niveau, vous devriez avoir le site en français et en anglais et voir l'ensemble de la page d'accueil. -celery -A pod.video_encode_transcript.importing_tasks worker -l INFO -Q importing --concurrency 1 --detach -n import_encode -celery -A pod.video_encode_transcript.importing_transcript_tasks worker -l INFO -Q importing_transcript --concurrency 1 --detach -n import_transcript python3 manage.py runserver 0.0.0.0:8080 --insecure sleep infinity diff --git a/pod/custom/settings_local_docker_full_test.py b/pod/custom/settings_local_docker_full_test.py new file mode 100644 index 0000000000..8e71aa0e1c --- /dev/null +++ b/pod/custom/settings_local_docker_full_test.py @@ -0,0 +1,75 @@ +DEBUG = True + +TEST_REMOTE_ENCODE = True +TEST_SETTINGS = True + +ALLOWED_HOSTS = ["*"] + +ADMINS = ( + ('Nicolas', 'nicolas@univ.fr'), +) + +USE_PODFILE = True +USE_NOTIFICATIONS = False +EMAIL_ON_ENCODING_COMPLETION = False +SECRET_KEY = 'A_CHANGER' + +# We specify here that we're using ES version 7\n +ES_VERSION = 7 +ES_URL = ['http://elasticsearch:9200/'] +CACHES = { + 'default': { + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'redis://redis:6379/3', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + }, + 'KEY_PREFIX': 'pod' + }, + 'select2': { + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'redis://redis:6379/2', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + }, + }, +} +SESSION_ENGINE = 'redis_sessions.session' +SESSION_REDIS = { + 'host': 'redis', + 'port': 6379, + 'db': 4, + 'prefix': 'session', + 'socket_timeout': 1, + 'retry_on_timeout': False, +} + +# Only in containerized environments +MIGRATION_MODULES = {'flatpages': 'pod.db_migrations'} + +# If DOCKER_ENV = full: activate encoding, transcription and remote xapi +USE_REMOTE_ENCODING_TRANSCODING = True +ENCODING_TRANSCODING_CELERY_BROKER_URL = 'redis://redis:6379/7' +POD_API_URL = "http://pod-back:8080/rest" +POD_API_TOKEN = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + +USE_TRANSCRIPTION = True +TRANSCRIPTION_TYPE = "WHISPER" +TRANSCRIPTION_MODEL_PARAM = { + 'WHISPER': { + 'fr': { + 'model': "small", + 'download_root': "/usr/src/app/transcription/whisper/", + }, + 'en': { + 'model': "small", + 'download_root': "/usr/src/app/transcription/whisper/", + } + } +} + +USE_XAPI_VIDEO = False +XAPI_CELERY_BROKER_URL = "redis://redis:6379/6" + +# for maximum console logging\n +LOGGING = {} diff --git a/pod/dressing/tests/test_utils.py b/pod/dressing/tests/test_utils.py index f48cdcb141..77bc36dcfe 100644 --- a/pod/dressing/tests/test_utils.py +++ b/pod/dressing/tests/test_utils.py @@ -5,29 +5,13 @@ from unittest.mock import patch from django.contrib.auth.models import User from pod.authentication.models import AccessGroup -from pod.dressing.utils import get_dressings, get_position_value, get_dressing_input +from pod.dressing.utils import get_dressings, get_dressing_input from pod.dressing.models import Dressing class DressingUtilitiesTests(unittest.TestCase): """TestCase for Esup-Pod dressing utilities.""" - def test_get_position_value(self): - """Test for the get_position_value function.""" - result = get_position_value("top_right", "720") - self.assertEqual(result, "overlay=main_w-overlay_w-36.0:36.0") - - result = get_position_value("top_left", "720") - self.assertEqual(result, "overlay=36.0:36.0") - - result = get_position_value("bottom_right", "720") - self.assertEqual(result, "overlay=main_w-overlay_w-36.0:main_h-overlay_h-36.0") - - result = get_position_value("bottom_left", "720") - self.assertEqual(result, "overlay=36.0:main_h-overlay_h-36.0") - - print(" ---> test_get_position_value: OK! --- DressingUtilsTest") - def test_get_dressing_input(self): """Test for the get_dressing_input function.""" dressing = Dressing(watermark=None, opening_credits=None, ending_credits=None) diff --git a/pod/dressing/utils.py b/pod/dressing/utils.py index 711b4680f3..bcd6728dda 100644 --- a/pod/dressing/utils.py +++ b/pod/dressing/utils.py @@ -6,28 +6,6 @@ from django.db.models import Q -def get_position_value(position: str, height: str) -> str: - """ - Obtain dimensions proportional to the video format. - - Args: - position (str): proprerty "position" of the dressing object. - height (str): height of the source video. - - Returns: - str: params for the ffmpeg command. - """ - height = str(float(height) * 0.05) - if position == "top_right": - return "overlay=main_w-overlay_w-" + height + ":" + height - elif position == "top_left": - return "overlay=" + height + ":" + height - elif position == "bottom_right": - return "overlay=main_w-overlay_w-" + height + ":main_h-overlay_h-" + height - elif position == "bottom_left": - return "overlay=" + height + ":main_h-overlay_h-" + height - - def get_dressing_input(dressing: Dressing, FFMPEG_DRESSING_INPUT: str) -> str: """ Obtain the files necessary for encoding a dressed video. diff --git a/pod/main/configuration.json b/pod/main/configuration.json index 1752dcb985..d60da9f343 100644 --- a/pod/main/configuration.json +++ b/pod/main/configuration.json @@ -3029,7 +3029,7 @@ "pod_version_end": "", "pod_version_init": "3.1.0" }, - "USE_DISTANT_ENCODING_TRANSCODING": { + "USE_REMOTE_ENCODING_TRANSCODING": { "default_value": false, "description": { "en": [ @@ -3041,7 +3041,37 @@ ] }, "pod_version_end": "", - "pod_version_init": "3.4.0" + "pod_version_init": "3.5.1" + }, + "POD_API_URL": { + "default_value": "", + "description": { + "en": [ + "Address of API rest to be called at the end of remote encoding or remote transcription.", + "Example : https://pod.univ.fr/rest/" + ], + "fr": [ + "Adresse de l'API rest a appeler en fin d'encodage distant ou de transcription à distance.", + "Exemple : https://pod.univ.fr/rest/" + ] + }, + "pod_version_end": "", + "pod_version_init": "3.5.1" + }, + "POD_API_TOKEN": { + "default_value": "", + "description": { + "en": [ + "Authentication token used for the call at the end of remote encoding or remote transcription.", + "To create it, go to Admin > Authentication token > token." + ], + "fr": [ + "Token d'authentification utilisé pour l'appel en fin d'encodage distant ou de transcription à distance.", + "Pour le créer, il faut aller dans la partie Admin > Jeton d'authentification > token." + ] + }, + "pod_version_end": "", + "pod_version_init": "3.5.1" }, "VIDEO_RENDITIONS": { "default_value": "[]", @@ -5052,7 +5082,7 @@ "Mise en place du mode PWA grâce à l'application Django-pwa", "Voici la configuration par défaut pour Pod, vous pouvez surcharger chaque variable dans votre fichier de configuration.", "PWA_APP_NAME = \"Pod\"", - "PWA_APP_DESCRIPTION = _(", + "PWA_APP_DESCRIPTION = (", " \"Pod is aimed at users of our institutions, by allowing the publication of \"", " \"videos in the fields of research (promotion of platforms, etc.), training \"", " \"(tutorials, distance training, student reports, etc.), institutional life (video \"", diff --git a/pod/main/forms.py b/pod/main/forms.py index c6ebdebd69..e9a1aa911e 100644 --- a/pod/main/forms.py +++ b/pod/main/forms.py @@ -21,6 +21,21 @@ ) +class DownloadFileForm(forms.Form): + """Manage "Download File" form.""" + filename = forms.FilePathField( + required=True, + path=settings.MEDIA_ROOT, + recursive=True, + allow_files=True, + allow_folders=False + ) + + def __init__(self, *args, **kwargs): + """Init download file form.""" + super(DownloadFileForm, self).__init__(*args, **kwargs) + + class ContactUsForm(forms.Form): """Manage "Contact us" form.""" diff --git a/pod/main/rest_router.py b/pod/main/rest_router.py index fbc551cc72..e03d33f756 100644 --- a/pod/main/rest_router.py +++ b/pod/main/rest_router.py @@ -75,7 +75,7 @@ urlpatterns = [ url(r"dublincore/$", video_views.DublinCoreView.as_view(), name="dublincore"), url( - r"launch_encode_view/$", + r"^launch_encode_view/$", encode_views.launch_encode_view, name="launch_encode_view", ), @@ -84,6 +84,11 @@ encode_views.store_remote_encoded_video, name="store_remote_encoded_video", ), + url( + r"store_remote_transcripted_video/$", + encode_views.store_remote_transcripted_video, + name="store_remote_transcripted_video", + ), url( r"accessgroups_set_users_by_name/$", auth_views.accessgroups_set_users_by_name, diff --git a/pod/main/settings.py b/pod/main/settings.py index b91776adc5..e87c4bd4ad 100644 --- a/pod/main/settings.py +++ b/pod/main/settings.py @@ -5,7 +5,6 @@ """ import os -from django.utils.translation import gettext_lazy as _ ## # flatpages @@ -312,7 +311,7 @@ # PWA PWA_APP_NAME = "Pod" -PWA_APP_DESCRIPTION = _( +PWA_APP_DESCRIPTION = ( "Pod is aimed at users of our institutions, by allowing the publication of " "videos in the fields of research (promotion of platforms, etc.), training " "(tutorials, distance training, student reports, etc.), institutional life (video " diff --git a/pod/main/static/video_test/video_test_encodage_transcription.webm b/pod/main/static/video_test/video_test_encodage_transcription.webm new file mode 100644 index 0000000000..6353c8255d Binary files /dev/null and b/pod/main/static/video_test/video_test_encodage_transcription.webm differ diff --git a/pod/main/test_settings.py b/pod/main/test_settings.py index 9528bfafc5..cce68fd4b4 100644 --- a/pod/main/test_settings.py +++ b/pod/main/test_settings.py @@ -21,9 +21,13 @@ ) USE_DOCKER = True path = "pod/custom/settings_local.py" +ES_URL = ["http://127.0.0.1:9200/"] +ES_VERSION = 6 if os.path.exists(path): _temp = __import__("pod.custom", globals(), locals(), ["settings_local"]) USE_DOCKER = getattr(_temp.settings_local, "USE_DOCKER", True) + ES_URL = getattr(_temp.settings_local, "ES_URL", ["http://127.0.0.1:9200/"]) + ES_VERSION = getattr(_temp.settings_local, "ES_VERSION", 6) for application in INSTALLED_APPS: if application.startswith("pod"): diff --git a/pod/main/tests/test_views.py b/pod/main/tests/test_views.py index eb30e3e119..85243b3593 100644 --- a/pod/main/tests/test_views.py +++ b/pod/main/tests/test_views.py @@ -9,18 +9,18 @@ from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User -from django.core.exceptions import PermissionDenied +from django.core.exceptions import PermissionDenied, SuspiciousOperation +from django.conf import settings from captcha.models import CaptchaStore from http import HTTPStatus from pod.main import context_processors from pod.main.models import Configuration -import tempfile -import os -import importlib from pod.playlist.models import Playlist - from pod.video.models import Type, Video +import os +import importlib + class MainViewsTestCase(TestCase): """Main views test cases.""" @@ -34,9 +34,9 @@ def setUp(self): User.objects.create(username="pod", password="podv3") print(" ---> SetUp of MainViewsTestCase: OK!") - @override_settings(MEDIA_ROOT=tempfile.gettempdir()) + @override_settings() def test_download_file(self): - """Test download folder.""" + """Test download file.""" self.client = Client() # GET method is used @@ -48,14 +48,16 @@ def test_download_file(self): response = self.client.post("/download/") self.assertRaises(PermissionDenied) # filename is properly set - temp_file = tempfile.NamedTemporaryFile() - response = self.client.post("/download/", {"filename": temp_file.name}) - self.assertEqual( - response["Content-Disposition"], - 'attachment; filename="%s"' % (os.path.basename(temp_file.name)), - ) - self.assertEqual(response.status_code, 200) - + response = self.client.post("/download/", {"filename": "/etc/passwd"}) + self.assertRaises(SuspiciousOperation) + filename = os.path.join(settings.MEDIA_ROOT, "test/test_file.txt") + if not os.path.exists(os.path.join(settings.MEDIA_ROOT, "test")): + os.mkdir(os.path.join(settings.MEDIA_ROOT, "test")) + f = open(filename, "w") + f.write("ok") + f.close() + response = self.client.post("/download/", {"filename": "test/test_file.txt"}) + self.assertTrue(response.status_code in [200, 400]) print(" ---> download_file of mainViewsTestCase: OK!") def test_contact_us(self): diff --git a/pod/main/views.py b/pod/main/views.py index 53aea195d0..5fcc78a048 100644 --- a/pod/main/views.py +++ b/pod/main/views.py @@ -17,6 +17,7 @@ import bleach from .forms import ContactUsForm, SUBJECT_CHOICES +from .forms import DownloadFileForm from django.shortcuts import render from django.contrib.sites.shortcuts import get_current_site from django.contrib import messages @@ -28,7 +29,7 @@ from django.utils.translation import ugettext_lazy as _ from django.shortcuts import redirect from django.contrib.auth.models import User -from django.core.exceptions import PermissionDenied +from django.core.exceptions import PermissionDenied, SuspiciousOperation from django.http import HttpResponse, HttpResponseBadRequest from django.http import JsonResponse from wsgiref.util import FileWrapper @@ -103,14 +104,22 @@ def in_maintenance(): def download_file(request): """Direct download of requested file.""" if request.POST and request.POST.get("filename"): - filename = os.path.join(settings.MEDIA_ROOT, request.POST["filename"]) - wrapper = FileWrapper(open(filename, "rb")) - response = HttpResponse(wrapper, content_type=mimetypes.guess_type(filename)[0]) - response["Content-Length"] = os.path.getsize(filename) - response["Content-Disposition"] = 'attachment; filename="%s"' % os.path.basename( - filename - ) - return response + filename = os.path.join(settings.MEDIA_ROOT, request.POST.get("filename")) + form = DownloadFileForm({"filename": filename}) + if form.is_valid(): + cleaned_filename = form.cleaned_data["filename"] + if os.path.isfile(cleaned_filename) and cleaned_filename.startswith(settings.MEDIA_ROOT): + wrapper = FileWrapper(open(filename, "rb")) + response = HttpResponse(wrapper, content_type=mimetypes.guess_type(filename)[0]) + response["Content-Length"] = os.path.getsize(filename) + response["Content-Disposition"] = 'attachment; filename="%s"' % os.path.basename( + filename + ) + return response + else: + raise SuspiciousOperation("file not exist or not in media") + else: + raise SuspiciousOperation("not valid file") else: raise PermissionDenied diff --git a/pod/video/models.py b/pod/video/models.py index 222fc80db7..92f5ff889f 100644 --- a/pod/video/models.py +++ b/pod/video/models.py @@ -1391,7 +1391,7 @@ def get_dublin_core(self): self.type.title, ", ".join( self.discipline.all() - .filter(sites=current_site) + .filter(site=current_site) .values_list("title", flat=True) ), ), diff --git a/pod/video_encode_transcript/Encoding_video.py b/pod/video_encode_transcript/Encoding_video.py index 2dc96d5076..8f0a7acb6a 100644 --- a/pod/video_encode_transcript/Encoding_video.py +++ b/pod/video_encode_transcript/Encoding_video.py @@ -6,10 +6,10 @@ from webvtt import WebVTT, Caption import argparse import unicodedata -from pod.dressing.utils import get_position_value, get_dressing_input if __name__ == "__main__": from encoding_utils import ( + get_dressing_position_value, get_info_from_video, get_list_rendition, launch_cmd, @@ -46,6 +46,7 @@ ) else: from .encoding_utils import ( + get_dressing_position_value, get_info_from_video, get_list_rendition, launch_cmd, @@ -174,9 +175,10 @@ class Encoding_video: error_encoding = False cutting_start = 0 cutting_stop = 0 - dressing = None + json_dressing = None + dressing_input = "" - def __init__(self, id=0, video_file="", start=0, stop=0, dressing=None): + def __init__(self, id=0, video_file="", start=0, stop=0, json_dressing=None, dressing_input=""): """Initialize a new Encoding_video object.""" self.id = id self.video_file = video_file @@ -199,7 +201,8 @@ def __init__(self, id=0, video_file="", start=0, stop=0, dressing=None): self.error_encoding = False self.cutting_start = start or 0 self.cutting_stop = stop or 0 - self.dressing = dressing or None + self.json_dressing = json_dressing + self.dressing_input = dressing_input def is_video(self) -> bool: """Check if current encoding correspond to a video.""" @@ -412,7 +415,7 @@ def get_dressing_command(self) -> str: "nb_threads": FFMPEG_NB_THREADS, } - dressing_command += get_dressing_input(self.dressing, FFMPEG_DRESSING_INPUT) + dressing_command += self.dressing_input dressing_command_filter = [] dressing_command_filter.append( FFMPEG_DRESSING_SCALE @@ -422,21 +425,21 @@ def get_dressing_command(self) -> str: "name": "vid", } ) - if self.dressing.watermark: + if self.json_dressing["watermark"]: dressing_command_params = "[video][0:a]" order_opening_credits = order_opening_credits + 1 name_out = "" - if self.dressing.opening_credits or self.dressing.ending_credits: + if self.json_dressing["opening_credits"] or self.json_dressing["ending_credits"]: name_out = "[video]" dressing_command_filter.append( FFMPEG_DRESSING_WATERMARK % { - "opacity": self.dressing.opacity / 100.0, - "position": get_position_value(self.dressing.position, height), + "opacity": self.json_dressing.opacity / 100.0, + "position": get_dressing_position_value(self.json_dressing["position"], height), "name_out": name_out, } ) - if self.dressing.opening_credits: + if self.json_dressing["opening_credits"]: order_opening_credits = order_opening_credits + 1 number_concat = number_concat + 1 dressing_command_params = ( @@ -450,7 +453,7 @@ def get_dressing_command(self) -> str: "name": "debut", } ) - if self.dressing.ending_credits: + if self.json_dressing["ending_credits"]: number_concat = number_concat + 1 dressing_command_params = ( dressing_command_params @@ -466,7 +469,7 @@ def get_dressing_command(self) -> str: "name": "fin", } ) - if self.dressing.opening_credits or self.dressing.ending_credits: + if self.json_dressing["opening_credits"] or self.json_dressing["ending_credits"]: dressing_command_filter.append( FFMPEG_DRESSING_CONCAT % { @@ -479,7 +482,7 @@ def get_dressing_command(self) -> str: "filter": ";".join(dressing_command_filter), } - if self.dressing.opening_credits or self.dressing.ending_credits: + if self.json_dressing["opening_credits"] or self.json_dressing["ending_credits"]: dressing_command += " -map '[v]' -map '[a]' " output_file = self.get_dressing_file() @@ -749,10 +752,7 @@ def get_subtitle_part(self): def export_to_json(self): data_to_dump = {} for attribute, value in self.__dict__.items(): - if attribute == "dressing" and value is not None: - data_to_dump[attribute] = value.to_json() - else: - data_to_dump[attribute] = value + data_to_dump[attribute] = value with open(self.output_dir + "/info_video.json", "w") as outfile: json.dump(data_to_dump, outfile, indent=2) @@ -766,7 +766,7 @@ def start_encode(self): self.start = time.ctime() self.create_output_dir() self.get_video_data() - if self.dressing is not None: + if self.json_dressing is not None: self.encode_video_dressing() print(self.id, self.video_file, self.duration) if self.is_video(): diff --git a/pod/video_encode_transcript/encode.py b/pod/video_encode_transcript/encode.py index ff0f773c74..6572148c66 100644 --- a/pod/video_encode_transcript/encode.py +++ b/pod/video_encode_transcript/encode.py @@ -10,7 +10,9 @@ from pod.cut.models import CutVideo from pod.dressing.models import Dressing +from pod.dressing.utils import get_dressing_input from pod.main.tasks import task_start_encode, task_start_encode_studio +from .encoding_settings import FFMPEG_DRESSING_INPUT from .utils import ( change_encoding_step, check_file, @@ -37,12 +39,16 @@ CELERY_TO_ENCODE = getattr(settings, "CELERY_TO_ENCODE", False) EMAIL_ON_ENCODING_COMPLETION = getattr(settings, "EMAIL_ON_ENCODING_COMPLETION", True) -USE_DISTANT_ENCODING_TRANSCODING = getattr( - settings, "USE_DISTANT_ENCODING_TRANSCODING", False +USE_REMOTE_ENCODING_TRANSCODING = getattr( + settings, "USE_REMOTE_ENCODING_TRANSCODING", False ) -if USE_DISTANT_ENCODING_TRANSCODING: +if USE_REMOTE_ENCODING_TRANSCODING: from .encoding_tasks import start_encoding_task +FFMPEG_DRESSING_INPUT = getattr( + settings, "FFMPEG_DRESSING_INPUT", FFMPEG_DRESSING_INPUT +) + # ########################################################################## # ENCODE VIDEO: THREAD TO LAUNCH ENCODE # ########################################################################## @@ -128,13 +134,23 @@ def encode_video(video_id: int) -> None: encoding_video.remove_old_data() change_encoding_step(video_id, 2, "start encoding") - if USE_DISTANT_ENCODING_TRANSCODING: + if USE_REMOTE_ENCODING_TRANSCODING: + dressing = None + dressing_input = "" + if Dressing.objects.filter(videos=video_to_encode).exists(): + dressing = Dressing.objects.get(videos=video_to_encode).to_json() + if dressing: + dressing_input = get_dressing_input( + dressing, + FFMPEG_DRESSING_INPUT + ) start_encoding_task.delay( encoding_video.id, encoding_video.video_file, encoding_video.cutting_start, encoding_video.cutting_stop, - encoding_video.dressing, + json_dressing=dressing, + dressing_input=dressing_input ) else: encoding_video.start_encode() @@ -164,20 +180,35 @@ def store_encoding_info(video_id: int, encoding_video: Encoding_video_model) -> def get_encoding_video(video_to_encode: Video) -> Encoding_video_model: """Get the encoding video object from video.""" dressing = None + dressing_input = "" if Dressing.objects.filter(videos=video_to_encode).exists(): - dressing = Dressing.objects.get(videos=video_to_encode) + dressing = Dressing.objects.get(videos=video_to_encode).to_json() + if dressing: + dressing_input = get_dressing_input( + dressing, + FFMPEG_DRESSING_INPUT + ) if CutVideo.objects.filter(video=video_to_encode).exists(): cut = CutVideo.objects.get(video=video_to_encode) cut_start = time_to_seconds(cut.start) cut_end = time_to_seconds(cut.end) encoding_video = Encoding_video_model( - video_to_encode.id, video_to_encode.video.path, cut_start, cut_end, dressing + video_to_encode.id, + video_to_encode.video.path, + cut_start, cut_end, + json_dressing=dressing, + dressing_input=dressing_input ) return encoding_video return Encoding_video_model( - video_to_encode.id, video_to_encode.video.path, 0, 0, dressing + video_to_encode.id, + video_to_encode.video.path, + 0, + 0, + json_dressing=dressing, + dressing_input=dressing_input ) diff --git a/pod/video_encode_transcript/encoding_tasks.py b/pod/video_encode_transcript/encoding_tasks.py index cfc6414965..e7e16cb7b9 100644 --- a/pod/video_encode_transcript/encoding_tasks.py +++ b/pod/video_encode_transcript/encoding_tasks.py @@ -5,6 +5,7 @@ # pip3 install redis==4.5.4 from celery import Celery import logging +import requests # call local settings directly # no need to load pod application to send statement @@ -13,12 +14,36 @@ except ImportError: from .. import settings as settings_local +EMAIL_HOST = getattr(settings_local, 'EMAIL_HOST', "") +DEFAULT_FROM_EMAIL = getattr(settings_local, 'DEFAULT_FROM_EMAIL', "") +ADMINS = getattr(settings_local, 'ADMINS', ()) +DEBUG = getattr(settings_local, 'DEBUG', True) +TEST_REMOTE_ENCODE = getattr(settings_local, "TEST_REMOTE_ENCODE", False) + +admins_email = [ad[1] for ad in ADMINS] + logger = logging.getLogger(__name__) +if DEBUG: + logger.setLevel(logging.DEBUG) + +smtp_handler = logging.handlers.SMTPHandler( + mailhost=EMAIL_HOST, + fromaddr=DEFAULT_FROM_EMAIL, + toaddrs=admins_email, + subject='[POD ENCODING] Encoding Log Mail' +) +if not TEST_REMOTE_ENCODE: + logger.addHandler(smtp_handler) ENCODING_TRANSCODING_CELERY_BROKER_URL = getattr( settings_local, "ENCODING_TRANSCODING_CELERY_BROKER_URL", "" ) - +POD_API_URL = getattr( + settings_local, "POD_API_URL", "" +) +POD_API_TOKEN = getattr( + settings_local, "POD_API_TOKEN", "" +) encoding_app = Celery("encoding_tasks", broker=ENCODING_TRANSCODING_CELERY_BROKER_URL) encoding_app.conf.task_routes = { "pod.video_encode_transcript.encoding_tasks.*": {"queue": "encoding"} @@ -27,22 +52,46 @@ # celery -A pod.video_encode_transcript.encoding_tasks worker -l INFO -Q encoding @encoding_app.task -def start_encoding_task(video_id, video_path, cut_start, cut_end, dressing): +def start_encoding_task( + video_id, video_path, cut_start, cut_end, json_dressing, dressing_input +): """Start the encoding of the video.""" print("Start the encoding of the video") from .Encoding_video import Encoding_video - from .importing_tasks import start_importing_task - print(video_id, video_path, cut_start, cut_end) - encoding_video = Encoding_video(video_id, video_path, cut_start, cut_end, dressing) + encoding_video = Encoding_video( + video_id, video_path, cut_start, cut_end, json_dressing, dressing_input + ) encoding_video.start_encode() print("End of the encoding of the video") - start_importing_task.delay( - encoding_video.start, - video_id, - video_path, - cut_start, - cut_end, - encoding_video.stop, - encoding_video.dressing, - ) + Headers = {"Authorization" : "Token %s" % POD_API_TOKEN} + url = POD_API_URL.strip("/") + "/store_remote_encoded_video/?id=%s" % video_id + data = { + "start": encoding_video.start, + "video_id": video_id, + "video_path": video_path, + "cut_start": cut_start, + "cut_end": cut_end, + "stop": encoding_video.stop, + "json_dressing": json_dressing, + "dressing_input": dressing_input + } + try: + response = requests.post(url, json=data, headers=Headers) + if response.status_code != 200: + msg = "Calling store remote encoding error : {} {}".format( + response.status_code, + response.reason + ) + logger.error(msg + "\n" + str(response.content)) + else: + logger.info("Call importing encoded task ok") + except ( + requests.exceptions.HTTPError, + requests.exceptions.ConnectionError, + requests.exceptions.InvalidURL, + requests.exceptions.Timeout + ) as exception: + msg = "Exception: {}".format(type(exception).__name__) + msg += "\nException message: {}".format(exception) + logger.error(msg) diff --git a/pod/video_encode_transcript/encoding_utils.py b/pod/video_encode_transcript/encoding_utils.py index 1b0b4aa76f..1331bbd37f 100644 --- a/pod/video_encode_transcript/encoding_utils.py +++ b/pod/video_encode_transcript/encoding_utils.py @@ -11,6 +11,36 @@ from encoding_settings import VIDEO_RENDITIONS +def sec_to_timestamp(total_seconds): + """Format time for webvtt caption.""" + hours = int(total_seconds / 3600) + minutes = int(total_seconds / 60 - hours * 60) + seconds = total_seconds - hours * 3600 - minutes * 60 + return "{:02d}:{:02d}:{:06.3f}".format(hours, minutes, seconds) + + +def get_dressing_position_value(position: str, height: str) -> str: + """ + Obtain dimensions proportional to the video format. + + Args: + position (str): proprerty "position" of the dressing object. + height (str): height of the source video. + + Returns: + str: params for the ffmpeg command. + """ + height = str(float(height) * 0.05) + if position == "top_right": + return "overlay=main_w-overlay_w-" + height + ":" + height + elif position == "top_left": + return "overlay=" + height + ":" + height + elif position == "bottom_right": + return "overlay=main_w-overlay_w-" + height + ":main_h-overlay_h-" + height + elif position == "bottom_left": + return "overlay=" + height + ":main_h-overlay_h-" + height + + def get_renditions(): try: from .models import VideoRendition diff --git a/pod/video_encode_transcript/importing_tasks.py b/pod/video_encode_transcript/importing_tasks.py deleted file mode 100644 index 8644d4171f..0000000000 --- a/pod/video_encode_transcript/importing_tasks.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Esup-Pod import video tasks.""" - -from celery import Celery - -try: - from ..custom import settings_local -except ImportError: - from .. import settings as settings_local -import os -import logging - -logger = logging.getLogger(__name__) - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pod.settings") - -ENCODING_TRANSCODING_CELERY_BROKER_URL = getattr( - settings_local, "ENCODING_TRANSCODING_CELERY_BROKER_URL", "" -) - -importing_app = Celery("importing_tasks", broker=ENCODING_TRANSCODING_CELERY_BROKER_URL) -importing_app.conf.task_routes = { - "pod.video_encode_transcript.importing_tasks.*": {"queue": "importing"} -} - - -# celery -A pod.video_encode_transcript.importing_tasks worker -l INFO -Q importing -@importing_app.task -def start_importing_task(start, video_id, video_path, cut_start, cut_end, stop): - """Start the import of the encoding of the video.""" - print("Start the importing of the video: %s" % video_id) - from .Encoding_video_model import Encoding_video_model - from .encode import store_encoding_info, end_of_encoding - - encoding_video = Encoding_video_model(video_id, video_path, cut_start, cut_end) - encoding_video.start = start - encoding_video.stop = stop - - final_video = store_encoding_info(video_id, encoding_video) - end_of_encoding(final_video) diff --git a/pod/video_encode_transcript/importing_transcript_tasks.py b/pod/video_encode_transcript/importing_transcript_tasks.py deleted file mode 100644 index 530481c289..0000000000 --- a/pod/video_encode_transcript/importing_transcript_tasks.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Esup-Pod import transcription tasks.""" - -from celery import Celery - -try: - from ..custom import settings_local -except ImportError: - from .. import settings as settings_local -import logging -import os -import webvtt - -logger = logging.getLogger(__name__) - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pod.settings") - -ENCODING_TRANSCODING_CELERY_BROKER_URL = getattr( - settings_local, "ENCODING_TRANSCODING_CELERY_BROKER_URL", "" -) - -importing_transcript_app = Celery( - "importing_transcript_tasks", broker=ENCODING_TRANSCODING_CELERY_BROKER_URL -) -importing_transcript_app.conf.task_routes = { - "pod.video_encode_transcript.importing_transcript_tasks.*": { - "queue": "importing_transcript" - } -} - - -# celery \ -# -A pod.video_encode_transcript.importing_transcript_tasks worker \ -# -l INFO -Q importing_transcript -@importing_transcript_app.task -def start_importing_transcript_task(video_id, msg, temp_vtt_file): - """Start the import of transcription of the video.""" - from pod.video.models import Video - from .transcript import save_vtt_and_notify - from ..main.settings import MEDIA_ROOT - - print("Start the import of transcription of the video: %s" % video_id) - print("temp_vtt_file: %s" % temp_vtt_file) - video_to_encode = Video.objects.get(id=video_id) - filename = os.path.basename(temp_vtt_file) - media_temp_dir = os.path.join(MEDIA_ROOT, "temp") - filepath = os.path.join(media_temp_dir, filename) - new_vtt = webvtt.read(filepath) - save_vtt_and_notify(video_to_encode, msg, new_vtt) - os.remove(filepath) diff --git a/pod/video_encode_transcript/management/__init__.py b/pod/video_encode_transcript/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pod/video_encode_transcript/management/commands/__init__.py b/pod/video_encode_transcript/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pod/video_encode_transcript/management/commands/test_encode_transcript.py b/pod/video_encode_transcript/management/commands/test_encode_transcript.py new file mode 100644 index 0000000000..aa58f1344d --- /dev/null +++ b/pod/video_encode_transcript/management/commands/test_encode_transcript.py @@ -0,0 +1,130 @@ +from django.core.management.base import BaseCommand, CommandError + +from django.conf import settings +from django.core.files.temp import NamedTemporaryFile +from django.contrib.auth.models import User +from rest_framework.authtoken.models import Token +from pod.video.models import Video, Type +from pod.video_encode_transcript import encode +from pod.video_encode_transcript.models import EncodingVideo +from pod.video_encode_transcript.models import PlaylistVideo +from pod.completion.models import Track + +import shutil +import os +import time + +VIDEO_TEST = "pod/main/static/video_test/video_test_encodage_transcription.webm" +ENCODE_VIDEO = getattr(settings, "ENCODE_VIDEO", "start_encode") +POD_API_URL = getattr( + settings, "POD_API_URL", "" +) +POD_API_TOKEN = getattr( + settings, "POD_API_TOKEN", "" +) +USE_TRANSCRIPTION = getattr(settings, "USE_TRANSCRIPTION", False) +if USE_TRANSCRIPTION: + from pod.video_encode_transcript import transcript + TRANSCRIPT_VIDEO = getattr(settings, "TRANSCRIPT_VIDEO", "start_transcript") + + +class Command(BaseCommand): + help = 'launch of video encoding and transcripting for video test : %s' % VIDEO_TEST + + def handle(self, *args, **options): + user, created = User.objects.update_or_create( + username="pod", + password="pod1234pod" + ) + user.is_staff = True + user.is_superuser = True + user.save() + # create token + if not Token.objects.filter(key=POD_API_TOKEN).exists(): + Token.objects.create(key=POD_API_TOKEN, user=user) + # owner1 = Owner.objects.get(user__username="pod") + video, created = Video.objects.update_or_create( + title="Video1", + owner=user, + # video="test.mp4", + type=Type.objects.get(id=1), + transcript="" + ) + tempfile = NamedTemporaryFile(delete=True) + video.video.save("test.mp4", tempfile) + dest = os.path.join(settings.MEDIA_ROOT, video.video.name) + shutil.copyfile(VIDEO_TEST, dest) + self.test_encoding(video) + self.test_transcripting(video) + print("\n -----> End of Encoding/transcripting video test") + + def test_transcripting(self, video): + print("\n ---> Start Transcripting video test") + if video.get_video_mp3() and not video.encoding_in_progress: + video.transcript = "fr" + video.save() + transcript_video = getattr(transcript, TRANSCRIPT_VIDEO) + transcript_video(video.id, threaded=False) + video.refresh_from_db() + n = 0 + while video.encoding_in_progress: + print("... Transcripting in progress : %s " % video.get_encoding_step) + video.refresh_from_db() + time.sleep(2) + n += 1 + if n > 60: + raise CommandError('Error while transcripting !!!') + video.refresh_from_db() + self.test_result_transcripting(video) + else: + raise CommandError('No mp3 found !!!') + print("\n ---> End of transcripting video test") + + def test_result_transcripting(self, video): + if not Track.objects.filter(video=video, lang="fr").exists(): + raise CommandError('Error while transcripting !!!') + + def test_encoding(self, video): + print("\n ---> Start Encoding video test") + encode_video = getattr(encode, ENCODE_VIDEO) + encode_video(video.id, threaded=False) + video.refresh_from_db() + n = 0 + while video.encoding_in_progress: + print("... Encoding in progress : %s " % video.get_encoding_step) + video.refresh_from_db() + time.sleep(2) + n += 1 + if n > 60: + raise CommandError('Error while encoding !!!') + video.refresh_from_db() + self.test_result_encoding(video) + print("\n ---> End of Encoding video test") + + def test_result_encoding(self, video): + list_mp2t = EncodingVideo.objects.filter( + video=video, encoding_format="video/mp2t" + ) + list_playlist_video = PlaylistVideo.objects.filter( + video=video, encoding_format="application/x-mpegURL" + ) + list_playlist_master = PlaylistVideo.objects.get( + name="playlist", + video=video, + encoding_format="application/x-mpegURL", + ) + list_mp4 = EncodingVideo.objects.filter( + video=video, encoding_format="video/mp4" + ) + if not len(list_mp2t) > 0: + raise CommandError("no video/mp2t found") + if not len(list_mp2t) + 1 == len(list_playlist_video): + raise CommandError("Error in playlist count") + if not list_playlist_master: + raise CommandError("No playlist master found") + if not len(list_mp4) > 0: + raise CommandError("No encoding mp4") + if not video.overview: + raise CommandError("No overview") + if not video.thumbnail: + raise CommandError("No thumbnails") diff --git a/pod/video_encode_transcript/rest_views.py b/pod/video_encode_transcript/rest_views.py index 44812e47ba..6684b38742 100644 --- a/pod/video_encode_transcript/rest_views.py +++ b/pod/video_encode_transcript/rest_views.py @@ -7,12 +7,21 @@ from rest_framework.decorators import action from rest_framework.decorators import api_view from rest_framework.response import Response + from django.shortcuts import get_object_or_404 +from django.views.decorators.csrf import csrf_exempt +from django.core.exceptions import SuspiciousOperation + +import json +import os +import webvtt USE_TRANSCRIPTION = getattr(settings, "USE_TRANSCRIPTION", False) if USE_TRANSCRIPTION: from pod.video_encode_transcript.transcript import start_transcript +MEDIA_ROOT = getattr(settings, "MEDIA_ROOT", "") + class VideoRenditionSerializer(serializers.HyperlinkedModelSerializer): class Meta: @@ -136,7 +145,7 @@ class PlaylistVideoViewSet(viewsets.ModelViewSet): serializer_class = PlaylistVideoSerializer -@api_view(["GET"]) +@api_view(['GET']) def launch_encode_view(request): """API view for launching video encoding.""" video = get_object_or_404(Video, slug=request.GET.get("slug")) @@ -161,10 +170,51 @@ def launch_transcript_view(request): return Response(VideoSerializer(instance=video, context={"request": request}).data) -@api_view(["GET"]) +@csrf_exempt +@api_view(["POST"]) def store_remote_encoded_video(request): """API view for storing remote encoded videos.""" + from .Encoding_video_model import Encoding_video_model + from .encode import store_encoding_info, end_of_encoding video_id = request.GET.get("id", 0) video = get_object_or_404(Video, id=video_id) # start_store_remote_encoding_video(video_id) + # check if video is encoding !!! + data = json.loads(request.body.decode("utf-8")) + if video.encoding_in_progress is False: + raise SuspiciousOperation("video not encoding in progress") + if str(video_id) != str(data["video_id"]): + raise SuspiciousOperation("different video id : %s - %s" % (video_id, data["video_id"])) + print("Start the importing of the video: %s" % video_id) + encoding_video = Encoding_video_model( + video_id, + data["video_path"], + data["cut_start"], + data["cut_end"] + ) + encoding_video.start = data["start"] + encoding_video.stop = data["stop"] + final_video = store_encoding_info(video_id, encoding_video) + end_of_encoding(final_video) + return Response(VideoSerializer(instance=video, context={"request": request}).data) + + +@csrf_exempt +@api_view(["POST"]) +def store_remote_transcripted_video(request): + """API view for storing remote transcripted videos.""" + from .transcript import save_vtt_and_notify + video_id = request.GET.get("id", 0) + video = get_object_or_404(Video, id=video_id) + # check if video is encoding !!! + data = json.loads(request.body.decode("utf-8")) + if str(video_id) != str(data["video_id"]): + raise SuspiciousOperation("different video id : %s - %s" % (video_id, data["video_id"])) + print("Start the import of transcription of the video: %s" % video_id) + filename = os.path.basename(data["temp_vtt_file"]) + media_temp_dir = os.path.join(MEDIA_ROOT, "temp") + filepath = os.path.join(media_temp_dir, filename) + new_vtt = webvtt.read(filepath) + save_vtt_and_notify(video, data["msg"], new_vtt) + os.remove(filepath) return Response(VideoSerializer(instance=video, context={"request": request}).data) diff --git a/pod/video_encode_transcript/tests/test_utils.py b/pod/video_encode_transcript/tests/test_utils.py new file mode 100644 index 0000000000..1248318a95 --- /dev/null +++ b/pod/video_encode_transcript/tests/test_utils.py @@ -0,0 +1,23 @@ +"""Unit tests for Esup-Pod dressing utilities.""" +import unittest +from ..encoding_utils import get_dressing_position_value + + +class EncodingUtilitiesTests(unittest.TestCase): + """TestCase for Esup-Pod encoding utilities.""" + + def test_dressing_position_value(self): + """Test for the get_position_value function.""" + result = get_dressing_position_value("top_right", "720") + self.assertEqual(result, "overlay=main_w-overlay_w-36.0:36.0") + + result = get_dressing_position_value("top_left", "720") + self.assertEqual(result, "overlay=36.0:36.0") + + result = get_dressing_position_value("bottom_right", "720") + self.assertEqual(result, "overlay=main_w-overlay_w-36.0:main_h-overlay_h-36.0") + + result = get_dressing_position_value("bottom_left", "720") + self.assertEqual(result, "overlay=36.0:main_h-overlay_h-36.0") + + print(" ---> get_dressing_position_value: OK! --- EncodginUtilsTest") diff --git a/pod/video_encode_transcript/transcript.py b/pod/video_encode_transcript/transcript.py index 5d30d43233..40e5b3dfb2 100644 --- a/pod/video_encode_transcript/transcript.py +++ b/pod/video_encode_transcript/transcript.py @@ -21,7 +21,9 @@ or importlib.util.find_spec("whisper") is not None ): from .transcript_model import start_transcripting - from .transcript_model import sec_to_timestamp + + +from .encoding_utils import sec_to_timestamp import os import time @@ -51,10 +53,10 @@ TRANSCRIPTION_NORMALIZE = getattr(settings, "TRANSCRIPTION_NORMALIZE", False) CELERY_TO_ENCODE = getattr(settings, "CELERY_TO_ENCODE", False) -USE_DISTANT_ENCODING_TRANSCODING = getattr( - settings, "USE_DISTANT_ENCODING_TRANSCODING", False +USE_REMOTE_ENCODING_TRANSCODING = getattr( + settings, "USE_REMOTE_ENCODING_TRANSCODING", False ) -if USE_DISTANT_ENCODING_TRANSCODING: +if USE_REMOTE_ENCODING_TRANSCODING: from .transcripting_tasks import start_transcripting_task log = logging.getLogger(__name__) @@ -102,7 +104,8 @@ def main_threaded_transcript(video_to_encode_id): change_encoding_step(video_to_encode_id, 5, "transcripting audio") video_to_encode = Video.objects.get(id=video_to_encode_id) - + video_to_encode.encoding_in_progress = True + video_to_encode.save() msg = "" lang = video_to_encode.transcript # check if TRANSCRIPTION_MODEL_PARAM [lang] exist @@ -123,7 +126,7 @@ def main_threaded_transcript(video_to_encode_id): send_email(msg, video_to_encode.id) else: mp3filepath = mp3file.path - if USE_DISTANT_ENCODING_TRANSCODING: + if USE_REMOTE_ENCODING_TRANSCODING: start_transcripting_task.delay( video_to_encode.id, mp3filepath, video_to_encode.duration, lang ) @@ -139,6 +142,8 @@ def save_vtt_and_notify(video_to_encode, msg, webvtt): """Call save vtt file function and notify by mail at the end.""" msg += saveVTT(video_to_encode, webvtt) change_encoding_step(video_to_encode.id, 0, "done") + video_to_encode.encoding_in_progress = False + video_to_encode.save() # envois mail fin transcription if EMAIL_ON_TRANSCRIPTING_COMPLETION: send_email_transcript(video_to_encode) diff --git a/pod/video_encode_transcript/transcript_model.py b/pod/video_encode_transcript/transcript_model.py index db2bd05009..f21e46a8d6 100644 --- a/pod/video_encode_transcript/transcript_model.py +++ b/pod/video_encode_transcript/transcript_model.py @@ -23,6 +23,8 @@ except ImportError: from .. import settings as settings_local +from .encoding_utils import sec_to_timestamp + DEBUG = getattr(settings_local, "DEBUG", False) TRANSCRIPTION_MODEL_PARAM = getattr(settings_local, "TRANSCRIPTION_MODEL_PARAM", False) @@ -458,14 +460,6 @@ def change_previous_end_caption(webvtt, start_caption): webvtt.captions[-1].end = sec_to_timestamp(start_caption) -def sec_to_timestamp(total_seconds): - """Format time for webvtt caption.""" - hours = int(total_seconds / 3600) - minutes = int(total_seconds / 60 - hours * 60) - seconds = total_seconds - hours * 3600 - minutes * 60 - return "{:02d}:{:02d}:{:06.3f}".format(hours, minutes, seconds) - - def get_text_caption(text_caption, last_word_added): """Get the text for a caption.""" try: diff --git a/pod/video_encode_transcript/transcripting_tasks.py b/pod/video_encode_transcript/transcripting_tasks.py index ee8ec4dcff..e323ec2f3e 100644 --- a/pod/video_encode_transcript/transcripting_tasks.py +++ b/pod/video_encode_transcript/transcripting_tasks.py @@ -7,7 +7,7 @@ from tempfile import NamedTemporaryFile import logging import os - +import requests # call local settings directly # no need to load pod application to send statement try: @@ -17,6 +17,33 @@ logger = logging.getLogger(__name__) +EMAIL_HOST = getattr(settings_local, 'EMAIL_HOST', "") +DEFAULT_FROM_EMAIL = getattr(settings_local, 'DEFAULT_FROM_EMAIL', "") +ADMINS = getattr(settings_local, 'ADMINS', ()) +DEBUG = getattr(settings_local, 'DEBUG', True) +TEST_REMOTE_ENCODE = getattr(settings_local, "TEST_REMOTE_ENCODE", False) + +admins_email = [ad[1] for ad in ADMINS] + +if DEBUG: + logger.setLevel(logging.DEBUG) + +smtp_handler = logging.handlers.SMTPHandler( + mailhost=EMAIL_HOST, + fromaddr=DEFAULT_FROM_EMAIL, + toaddrs=admins_email, + subject='[POD ENCODING] Encoding Log Mail' +) +if not TEST_REMOTE_ENCODE: + logger.addHandler(smtp_handler) + +POD_API_URL = getattr( + settings_local, "POD_API_URL", "" +) +POD_API_TOKEN = getattr( + settings_local, "POD_API_TOKEN", "" +) + ENCODING_TRANSCODING_CELERY_BROKER_URL = getattr( settings_local, "ENCODING_TRANSCODING_CELERY_BROKER_URL", "" ) @@ -37,7 +64,6 @@ def start_transcripting_task(video_id, mp3filepath, duration, lang): """Start the transcripting of the video.""" from .transcript_model import start_transcripting - from .importing_transcript_tasks import start_importing_transcript_task from ..main.settings import MEDIA_ROOT print("Start the transcripting of the video %s" % video_id) @@ -49,4 +75,30 @@ def start_transcripting_task(video_id, mp3filepath, duration, lang): os.mkdir(media_temp_dir) temp_vtt_file = NamedTemporaryFile(dir=media_temp_dir, delete=False, suffix=".vtt") text_webvtt.save(temp_vtt_file.name) - start_importing_transcript_task.delay(video_id, msg, temp_vtt_file.name) + print("End of the transcoding of the video") + Headers = {"Authorization" : "Token %s" % POD_API_TOKEN} + url = POD_API_URL.strip("/") + "/store_remote_transcripted_video/?id=%s" % video_id + data = { + "video_id": video_id, + "msg": msg, + "temp_vtt_file": temp_vtt_file.name + } + try: + response = requests.post(url, json=data, headers=Headers) + if response.status_code != 200: + msg = "Calling store remote transcoding error : {} {}".format( + response.status_code, + response.reason + ) + logger.error(msg + "\n" + str(response.content)) + else: + logger.info("Call importing transcript task ok") + except ( + requests.exceptions.HTTPError, + requests.exceptions.ConnectionError, + requests.exceptions.InvalidURL, + requests.exceptions.Timeout + ) as exception: + msg = "Exception: {}".format(type(exception).__name__) + msg += "\nException message: {}".format(exception) + logger.error(msg) diff --git a/pod/video_search/tests/test_utils.py b/pod/video_search/tests/test_utils.py new file mode 100644 index 0000000000..5d657d4e53 --- /dev/null +++ b/pod/video_search/tests/test_utils.py @@ -0,0 +1,35 @@ +"""Unit tests for Esup-Pod video search utilities.""" + +from django.test import TestCase +from django.contrib.auth.models import User + +from pod.video.models import Video, Type +from ..utils import index_es, delete_es + + +class VideoSearchTestUtils(TestCase): + """TestCase for Esup-Pod video utilities.""" + + fixtures = [ + "initial_data.json", + ] + + def setUp(self): + """Set up required objects for next tests.""" + self.user = User.objects.create(username="pod", password="pod1234pod") + self.v = Video.objects.create( + title="Video1", + owner=self.user, + video="test.mp4", + is_draft=False, + type=Type.objects.get(id=1), + ) + + def test_index_and_delete_es(self): + res = index_es(self.v) + self.assertTrue(res['result'] in ['created', 'updated']) + self.assertEqual(res['_id'], str(self.v.id)) + delete = delete_es(self.v) + self.assertEqual(delete['result'], 'deleted') + self.assertEqual(delete['_id'], str(self.v.id)) + print("--> test_index_and_delete_es ok ! ")