Scrolling_Subtitles_On_Video_using_Python/Scrolling_Subtitles_On_Video_using_Python.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ramsrigouthamg/Supertranslate.ai/blob/main/Scrolling_Subtitles_On_Video_using_Python/Scrolling_Subtitles_On_Video_using_Python.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CCjyk1zgGeFK"
      },
      "source": [
        "Code and files for this notebook: https://github.com/ramsrigouthamg/Supertranslate.ai/tree/main/Scrolling_Subtitles_On_Video_using_Python\n",
        "\n",
        "Author: [Ramsri Goutham](https://twitter.com/ramsri_goutham)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tKs_aVFjb1iP"
      },
      "outputs": [],
      "source": [
        "!pip install --quiet ipython-autotime"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DxDG58qJc6sX"
      },
      "outputs": [],
      "source": [
        "%load_ext autotime"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XKf1whNlpykK"
      },
      "source": [
        "## Step1: Download video or upload the video in mp4 format that you want to convert to fancy word-level subtitles (audiogram)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7GCbvWXzXeUv"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "\n",
        "mp4videoURL = \"https://github.com/ramsrigouthamg/Supertranslate.ai/raw/main/Descript_like_wordhighlights_subtitles/Intro.mp4\"\n",
        "videofilename = mp4videoURL.split('/')[-1]\n",
        "print (videofilename)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HpTtfXWmYnIR"
      },
      "outputs": [],
      "source": [
        "import urllib.request\n",
        "urllib.request.urlretrieve(mp4videoURL, videofilename)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "01R02XojV56i"
      },
      "source": [
        "## Step2: Extract audio (mp3) from the video file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kamWr_5paBJU"
      },
      "outputs": [],
      "source": [
        "!pip install ffmpeg-python==0.2.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1KYai-y0aT0p"
      },
      "outputs": [],
      "source": [
        "import ffmpeg\n",
        "\n",
        "audiofilename = videofilename.replace(\".mp4\",'.mp3')\n",
        "\n",
        "# Create the ffmpeg input stream\n",
        "input_stream = ffmpeg.input(videofilename)\n",
        "\n",
        "# Extract the audio stream from the input stream\n",
        "audio = input_stream.audio\n",
        "\n",
        "# Save the audio stream as an MP3 file\n",
        "output_stream = ffmpeg.output(audio, audiofilename)\n",
        "\n",
        "# Overwrite output file if it already exists\n",
        "output_stream = ffmpeg.overwrite_output(output_stream)\n",
        "\n",
        "ffmpeg.run(output_stream)\n",
        "\n",
        "print (audiofilename)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hyc89jXya7F7"
      },
      "outputs": [],
      "source": [
        "from IPython.display import Audio\n",
        "\n",
        "Audio(audiofilename)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_E1kn2aYWQRX"
      },
      "source": [
        "## Step3: Get word-level transcription (speech-to-text) from audio using OpenAI's Whisper"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nz1xM8J3237t"
      },
      "source": [
        "https://github.com/guillaumekln/faster-whisper"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xWFHdNEtbGeH"
      },
      "outputs": [],
      "source": [
        "!pip install faster-whisper==0.7.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4mKWjv8fTBLU"
      },
      "outputs": [],
      "source": [
        "from faster_whisper import WhisperModel\n",
        "\n",
        "model_size = \"medium\"\n",
        "model = WhisperModel(model_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x8Oxt72ZTgAC"
      },
      "outputs": [],
      "source": [
        "segments, info = model.transcribe(audiofilename, word_timestamps=True)\n",
        "segments = list(segments)  # The transcription will actually run here.\n",
        "for segment in segments:\n",
        "    for word in segment.words:\n",
        "        print(\"[%.2fs -> %.2fs] %s\" % (word.start, word.end, word.word))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kHaUPOH1eIZ8"
      },
      "outputs": [],
      "source": [
        "wordlevel_info = []\n",
        "\n",
        "for segment in segments:\n",
        "    for word in segment.words:\n",
        "      wordlevel_info.append({'word':word.word,'start':word.start,'end':word.end})\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kHostLbAfaO2"
      },
      "outputs": [],
      "source": [
        "wordlevel_info"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xl_GzTu-WWE5"
      },
      "source": [
        "## Step4: Create a simple UI to edit any errors"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aoTO9pXTWctz"
      },
      "outputs": [],
      "source": [
        "from IPython.display import YouTubeVideo\n",
        "from ipywidgets import widgets\n",
        "from IPython.display import display"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kYTK9QsTWeOa"
      },
      "outputs": [],
      "source": [
        "# Create a table to display the wordlevel_info\n",
        "table_columns = ['Start', 'End', 'Word']\n",
        "# Create the table rows\n",
        "table_rows = []\n",
        "\n",
        "for word_info in wordlevel_info:\n",
        "    start_widget = widgets.Text(value=str(word_info['start']))\n",
        "    end_widget = widgets.Text(value=str(word_info['end']))\n",
        "    word_widget = widgets.Text(value=word_info['word'])\n",
        "    row_widgets = widgets.HBox([start_widget, end_widget, word_widget])\n",
        "    table_rows.append(row_widgets)\n",
        "\n",
        "\n",
        "# Create the update button\n",
        "button = widgets.Button(description='Update')\n",
        "\n",
        "# Create the output area for displaying the updated wordlevel_info\n",
        "output_area = widgets.Output()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eTFNJVawWu3V"
      },
      "outputs": [],
      "source": [
        "# Create a new variable to store the modified wordlevel_info\n",
        "modified_wordlevel_info = []\n",
        "\n",
        "def update_wordlevel_info(event):\n",
        "    with output_area:\n",
        "        output_area.clear_output()\n",
        "        updated_wordlevel_info = []\n",
        "\n",
        "        for row_widgets in table_rows:\n",
        "            start = float(row_widgets.children[0].value)  # Convert start to float\n",
        "            end = float(row_widgets.children[1].value)  # Convert end to float\n",
        "            word = row_widgets.children[2].value\n",
        "            word = word.strip()\n",
        "            updated_wordlevel_info.append({'start': start, 'end': end, 'word': word})\n",
        "\n",
        "        # Print the updated wordlevel_info\n",
        "        print(updated_wordlevel_info)\n",
        "\n",
        "        # Assign updated_wordlevel_info to modified_wordlevel_info\n",
        "        global modified_wordlevel_info\n",
        "        modified_wordlevel_info = updated_wordlevel_info\n",
        "\n",
        "button.on_click(update_wordlevel_info)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k5YTobPPWwa1"
      },
      "outputs": [],
      "source": [
        "# Create the table widget\n",
        "# Create the table widget\n",
        "# Create the table widget\n",
        "table_widget = widgets.VBox(table_rows)\n",
        "\n",
        "# Display the UI elements\n",
        "display(table_widget, button, output_area)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4ygqhRm5ZYVu"
      },
      "outputs": [],
      "source": [
        "modified_wordlevel_info"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4FVtpppnWyPL"
      },
      "source": [
        "## Step5: Store word-level timestamps into JSON file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yvIVz0oTkdRb"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "with open('data.json', 'w') as f:\n",
        "    json.dump(modified_wordlevel_info, f,indent=4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ov-M27CrZSpP"
      },
      "source": [
        "## Step6: Correct and save JSON file if you need to merge any words into phrases. Load the new edited JSON file with new word-level timestamps"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q19dFuXkn6VD"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "\n",
        "with open('data.json', 'r') as f:\n",
        "    wordlevel_info_modified = json.load(f)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5gvj3OdfoLEM"
      },
      "outputs": [],
      "source": [
        "wordlevel_info_modified"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uni9XkxaXQzi"
      },
      "source": [
        "## Step6: Convert word-level timestamps JSON to line-level timestamps JSON. We want to display one line at a time and highlight words in it as they are spoken so we do this transformation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y0aSTtzApekX"
      },
      "outputs": [],
      "source": [
        "def split_text_into_lines(data):\n",
        "\n",
        "    MaxChars = 30\n",
        "    #maxduration in seconds\n",
        "    MaxDuration = 2.5\n",
        "    #Split if nothing is spoken (gap) for these many seconds\n",
        "    MaxGap = 1.5\n",
        "\n",
        "    subtitles = []\n",
        "    line = []\n",
        "    line_duration = 0\n",
        "    line_chars = 0\n",
        "\n",
        "\n",
        "    for idx,word_data in enumerate(data):\n",
        "        word = word_data[\"word\"]\n",
        "        start = word_data[\"start\"]\n",
        "        end = word_data[\"end\"]\n",
        "\n",
        "        line.append(word_data)\n",
        "        line_duration += end - start\n",
        "\n",
        "        temp = \" \".join(item[\"word\"] for item in line)\n",
        "\n",
        "\n",
        "        # Check if adding a new word exceeds the maximum character count or duration\n",
        "        new_line_chars = len(temp)\n",
        "\n",
        "        duration_exceeded = line_duration > MaxDuration\n",
        "        chars_exceeded = new_line_chars > MaxChars\n",
        "        if idx>0:\n",
        "          gap = word_data['start'] - data[idx-1]['end']\n",
        "          # print (word,start,end,gap)\n",
        "          maxgap_exceeded = gap > MaxGap\n",
        "        else:\n",
        "          maxgap_exceeded = False\n",
        "\n",
        "\n",
        "        if duration_exceeded or chars_exceeded or maxgap_exceeded:\n",
        "            if line:\n",
        "                subtitle_line = {\n",
        "                    \"word\": \" \".join(item[\"word\"] for item in line),\n",
        "                    \"start\": line[0][\"start\"],\n",
        "                    \"end\": line[-1][\"end\"],\n",
        "                    \"textcontents\": line\n",
        "                }\n",
        "                subtitles.append(subtitle_line)\n",
        "                line = []\n",
        "                line_duration = 0\n",
        "                line_chars = 0\n",
        "\n",
        "\n",
        "    if line:\n",
        "        subtitle_line = {\n",
        "            \"word\": \" \".join(item[\"word\"] for item in line),\n",
        "            \"start\": line[0][\"start\"],\n",
        "            \"end\": line[-1][\"end\"],\n",
        "            \"textcontents\": line\n",
        "        }\n",
        "        subtitles.append(subtitle_line)\n",
        "\n",
        "    return subtitles"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GUvxaeTdtXpl"
      },
      "outputs": [],
      "source": [
        "linelevel_subtitles = split_text_into_lines(wordlevel_info_modified)\n",
        "print (linelevel_subtitles)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W3HG0zHhu9vO"
      },
      "outputs": [],
      "source": [
        "for line in linelevel_subtitles:\n",
        "  json_str = json.dumps(line, indent=4)\n",
        "  print(json_str)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vCymtgRfXlHT"
      },
      "source": [
        "## Step7: Use Moviepy to create an audiogram with word-level highlights as they are spoken"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0M04nc-UkVQk"
      },
      "outputs": [],
      "source": [
        "!pip install moviepy==2.0.0.dev2\n",
        "!pip install imageio==2.25.1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "olhtrcx0k0W3"
      },
      "outputs": [],
      "source": [
        "!apt install imagemagick"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P44vCYP4k4Et"
      },
      "outputs": [],
      "source": [
        "!cat /etc/ImageMagick-6/policy.xml | sed 's/none/read,write/g'> /etc/ImageMagick-6/policy.xml"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YsBTKCm4lGFk"
      },
      "outputs": [],
      "source": [
        "from moviepy.editor import TextClip, CompositeVideoClip, ColorClip\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def create_caption(textJSON, framesize,font = \"Helvetica\",color='white', highlight_color='yellow',stroke_color='black',stroke_width=1.5):\n",
        "    wordcount = len(textJSON['textcontents'])\n",
        "    full_duration = textJSON['end']-textJSON['start']\n",
        "\n",
        "    word_clips = []\n",
        "    xy_textclips_positions =[]\n",
        "\n",
        "    x_pos = 0\n",
        "    y_pos = 0\n",
        "    line_width = 0  # Total width of words in the current line\n",
        "    frame_width = framesize[0]\n",
        "    frame_height = framesize[1]\n",
        "\n",
        "    x_buffer = frame_width*1/10\n",
        "\n",
        "    max_line_width = frame_width - 2 * (x_buffer)\n",
        "\n",
        "    fontsize = int(frame_height * 0.075) #7.5 percent of video height\n",
        "\n",
        "    space_width = \"\"\n",
        "    space_height = \"\"\n",
        "\n",
        "    for index,wordJSON in enumerate(textJSON['textcontents']):\n",
        "      duration = wordJSON['end']-wordJSON['start']\n",
        "      word_clip = TextClip(wordJSON['word'], font = font,fontsize=fontsize, color=color,stroke_color=stroke_color,stroke_width=stroke_width).set_start(textJSON['start']).set_duration(full_duration)\n",
        "      word_clip_space = TextClip(\" \", font = font,fontsize=fontsize, color=color).set_start(textJSON['start']).set_duration(full_duration)\n",
        "      word_width, word_height = word_clip.size\n",
        "      space_width,space_height = word_clip_space.size\n",
        "      if line_width + word_width+ space_width <= max_line_width:\n",
        "            # Store info of each word_clip created\n",
        "            xy_textclips_positions.append({\n",
        "                \"x_pos\":x_pos,\n",
        "                \"y_pos\": y_pos,\n",
        "                \"width\" : word_width,\n",
        "                \"height\" : word_height,\n",
        "                \"word\": wordJSON['word'],\n",
        "                \"start\": wordJSON['start'],\n",
        "                \"end\": wordJSON['end'],\n",
        "                \"duration\": duration\n",
        "            })\n",
        "\n",
        "            word_clip = word_clip.set_position((x_pos, y_pos))\n",
        "            word_clip_space = word_clip_space.set_position((x_pos+ word_width, y_pos))\n",
        "\n",
        "            x_pos = x_pos + word_width+ space_width\n",
        "            line_width = line_width+ word_width + space_width\n",
        "      else:\n",
        "            # Move to the next line\n",
        "            x_pos = 0\n",
        "            y_pos = y_pos+ word_height+10\n",
        "            line_width = word_width + space_width\n",
        "\n",
        "            # Store info of each word_clip created\n",
        "            xy_textclips_positions.append({\n",
        "                \"x_pos\":x_pos,\n",
        "                \"y_pos\": y_pos,\n",
        "                \"width\" : word_width,\n",
        "                \"height\" : word_height,\n",
        "                \"word\": wordJSON['word'],\n",
        "                \"start\": wordJSON['start'],\n",
        "                \"end\": wordJSON['end'],\n",
        "                \"duration\": duration\n",
        "            })\n",
        "\n",
        "            word_clip = word_clip.set_position((x_pos, y_pos))\n",
        "            word_clip_space = word_clip_space.set_position((x_pos+ word_width , y_pos))\n",
        "            x_pos = word_width + space_width\n",
        "\n",
        "\n",
        "      word_clips.append(word_clip)\n",
        "      word_clips.append(word_clip_space)\n",
        "\n",
        "\n",
        "    for highlight_word in xy_textclips_positions:\n",
        "\n",
        "      word_clip_highlight = TextClip(highlight_word['word'], font = font,fontsize=fontsize, color=highlight_color,stroke_color=stroke_color,stroke_width=stroke_width).set_start(highlight_word['start']).set_duration(highlight_word['duration'])\n",
        "      word_clip_highlight = word_clip_highlight.set_position((highlight_word['x_pos'], highlight_word['y_pos']))\n",
        "      word_clips.append(word_clip_highlight)\n",
        "\n",
        "    return word_clips,xy_textclips_positions\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from moviepy.editor import TextClip, CompositeVideoClip, concatenate_videoclips,VideoFileClip, ColorClip\n",
        "input_video = VideoFileClip(videofilename)\n",
        "frame_size = input_video.size\n",
        "\n",
        "all_linelevel_splits=[]\n",
        "\n",
        "for line in linelevel_subtitles:\n",
        "  out_clips,positions = create_caption(line,frame_size)\n",
        "\n",
        "  max_width = 0\n",
        "  max_height = 0\n",
        "\n",
        "  for position in positions:\n",
        "    # print (out_clip.pos)\n",
        "    # break\n",
        "    x_pos, y_pos = position['x_pos'],position['y_pos']\n",
        "    width, height = position['width'],position['height']\n",
        "\n",
        "    max_width = max(max_width, x_pos + width)\n",
        "    max_height = max(max_height, y_pos + height)\n",
        "\n",
        "  color_clip = ColorClip(size=(int(max_width*1.1), int(max_height*1.1)),\n",
        "                       color=(64, 64, 64))\n",
        "  color_clip = color_clip.set_opacity(.6)\n",
        "  color_clip = color_clip.set_start(line['start']).set_duration(line['end']-line['start'])\n",
        "\n",
        "  # centered_clips = [each.set_position('center') for each in out_clips]\n",
        "\n",
        "  clip_to_overlay = CompositeVideoClip([color_clip]+ out_clips)\n",
        "  clip_to_overlay = clip_to_overlay.set_position(\"bottom\")\n",
        "\n",
        "\n",
        "  all_linelevel_splits.append(clip_to_overlay)\n",
        "\n",
        "input_video_duration = input_video.duration\n",
        "\n",
        "\n",
        "final_video = CompositeVideoClip([input_video] + all_linelevel_splits)\n",
        "\n",
        "# Set the audio of the final video to be the same as the input video\n",
        "final_video = final_video.set_audio(input_video.audio)\n",
        "\n",
        "# Save the final clip as a video file with the audio included\n",
        "final_video.write_videofile(\"output.mp4\", fps=24, codec=\"libx264\", audio_codec=\"aac\")"
      ],
      "metadata": {
        "id": "QHnz5jJUqiuw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gos0diSTYAX4"
      },
      "source": [
        "## Step8: Visualize the final Descript like audiogram video"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Em5Cp-XEowrt"
      },
      "outputs": [],
      "source": [
        "from IPython.display import HTML\n",
        "from base64 import b64encode\n",
        "mp4 = open(\"output.mp4\",'rb').read()\n",
        "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n",
        "HTML(\"\"\"\n",
        "<video width=600 controls>\n",
        "      <source src=\"%s\" type=\"video/mp4\">\n",
        "</video>\n",
        "\"\"\" % data_url)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NBh4bNO9_aOU"
      },
      "source": [
        "## Extra - Check all fonts available"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EuVpLc6t_fNZ"
      },
      "outputs": [],
      "source": [
        "from moviepy.editor import TextClip\n",
        "\n",
        "# Create a dummy TextClip object\n",
        "dummy_clip = TextClip('Dummy Text')\n",
        "\n",
        "# Get the available fonts\n",
        "available_fonts = dummy_clip.list('font')\n",
        "\n",
        "# Print the available fonts\n",
        "for font in available_fonts:\n",
        "    print(font)\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPp5LK871Mnd9i+MkDmMz33",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}