examples/load.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sign Language Datasets Example",
      "private_outputs": true,
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sign-language-processing/datasets/blob/master/examples/load.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ov6fuFwGjlsy"
      },
      "source": [
        "%%capture\n",
        "! pip install git+https://github.com/sign-language-processing/datasets.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4PZsi6pPp9j"
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "import sign_language_datasets.datasets\n",
        "from sign_language_datasets.datasets.config import SignDatasetConfig\n",
        "\n",
        "import itertools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PKGZ4JXCZmSE"
      },
      "source": [
        "# RWTH Phoenix 2014 T"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8wU1Q4URqRBE"
      },
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"3.0.0\", include_video=False)\n",
        "rwth_phoenix2014_t = tfds.load(name='rwth_phoenix2014_t', builder_kwargs=dict(config=config))\n",
        "\n",
        "for datum in itertools.islice(rwth_phoenix2014_t[\"train\"], 0, 10):\n",
        "  print(datum['gloss'].numpy().decode('utf-8'))\n",
        "  print(datum['text'].numpy().decode('utf-8'))\n",
        "  print()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6iBwM9lTzS6"
      },
      "source": [
        "# Dicta Sign"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, we set up HamNoSys in Google Colab for it to be visible."
      ],
      "metadata": {
        "id": "hEjOCQ_VYeUG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import HTML\n",
        "\n",
        "font_url = 'https://ctan.math.illinois.edu/fonts/hamnosys/HamNoSysUnicode.ttf'\n",
        "\n",
        "# Define the custom CSS to inject\n",
        "custom_css = f\"\"\"\n",
        "@font-face {{\n",
        "    font-family: 'HamNoSysUnicode';\n",
        "    src: url('{font_url}') format('truetype');\n",
        "}}\n",
        "\"\"\"\n",
        "\n",
        "# Display the HTML with the custom CSS\n",
        "HTML(f'<style>{custom_css}</style>')"
      ],
      "metadata": {
        "id": "HN7S2m-BYc4s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EQWUAgpVT0bK"
      },
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None)\n",
        "dicta_sign = tfds.load(name='dicta_sign', builder_kwargs={\"config\": config})\n",
        "\n",
        "for datum in itertools.islice(dicta_sign[\"train\"], 0, 10):\n",
        "    hamnosys_text = datum['hamnosys'].numpy().decode('utf-8')\n",
        "    plain_text = datum['text'].numpy().decode('utf-8')\n",
        "    display(HTML(f'<span style=\"font-family:HamNoSysUnicode;\">{hamnosys_text}</span> {plain_text}'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OcIs13W6TfWz"
      },
      "source": [
        "# ChicagoFSWild+"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o1X1kIgoTfec"
      },
      "source": [
        "# Version 2.0.0 is ChicagoFSWild+, 1.0.0 is ChicagoFSWild\n",
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"2.0.0\", include_video=False)\n",
        "chicagofswild = tfds.load(name='chicago_fs_wild', builder_kwargs=dict(config=config))\n",
        "\n",
        "for datum in itertools.islice(chicagofswild[\"train\"], 0, 10):\n",
        "  print(datum['text'].numpy().decode('utf-8'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XK7jyOOtYv_P"
      },
      "source": [
        "# AUTSL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dfZnI9K8YxfJ"
      },
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n",
        "autsl = tfds.load(name='autsl', builder_kwargs={\"config\": config})\n",
        "\n",
        "for datum in itertools.islice(autsl[\"train\"], 0, 10):\n",
        "  print(datum['id'].numpy().decode('utf-8'), datum['gloss_id'].numpy())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rykmI68x3E07"
      },
      "source": [
        "# SignBank"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12XcWfeg21kE"
      },
      "source": [
        "signbank = tfds.load(name='sign_bank')\n",
        "\n",
        "for datum in itertools.islice(signbank[\"train\"], 0, 10):\n",
        "  print(datum['id'].numpy().decode('utf-8'), datum['sign_writing'].numpy().decode('utf-8'), [f.decode('utf-8') for f in datum['terms'].numpy()])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "biXjC80j17n1"
      },
      "source": [
        "# SignTyp (https://signtyp.uconn.edu/signpuddle/index.php?ui=1&sgn=9032)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dVgbyUIg165c"
      },
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, extra={\"PHPSESSID\": \"hj9co07ct7f5noq529no9u09l4\"})\n",
        "signtyp = tfds.load(name='sign_typ', builder_kwargs=dict(config=config))\n",
        "\n",
        "for datum in itertools.islice(signtyp[\"train\"], 0, 10):\n",
        "  print(datum['video'].numpy().decode('utf-8'), datum['sign_writing'].numpy().decode('utf-8'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yOLfw9-z2qK7"
      },
      "source": [
        "# Sign2Mint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X96ogmu_22zv"
      },
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n",
        "sign2mint = tfds.load(name='sign2_mint', builder_kwargs={\"config\": config})\n",
        "\n",
        "for datum in itertools.islice(sign2mint[\"train\"], 0, 10):\n",
        "  print(datum['fachbegriff'].numpy().decode('utf-8'), datum['video'].numpy().decode('utf-8'), datum['gebaerdenschrift']['url'].numpy().decode('utf-8'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jnf4AaX936w4"
      },
      "source": [
        "# SWOJS Glossário"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "shQxQtQP359y"
      },
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n",
        "swojs_glossario = tfds.load(name='swojs_glossario', builder_kwargs={\"config\": config})\n",
        "\n",
        "def decode(tl):\n",
        "  return list(map(lambda t: t.decode('utf-8'), tl.numpy()))\n",
        "\n",
        "for datum in itertools.islice(swojs_glossario[\"train\"], 0, 10):\n",
        "  print(decode(datum['sign_writing']), datum['video'].numpy().decode('utf-8'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pNJdG7ExZugh"
      },
      "source": [
        "# DGS Corpus"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVjrhsbtbWbX"
      },
      "source": [
        "%%capture\n",
        "! pip install pympi-ling"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Document Level example (Long videos)"
      ],
      "metadata": {
        "collapsed": false,
        "id": "G1ZxOKNKX2Q3"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1XvMTs9Zvx4"
      },
      "source": [
        "from sign_language_datasets.datasets.dgs_corpus import DgsCorpusConfig\n",
        "\n",
        "config = DgsCorpusConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None)\n",
        "dgs_corpus = tfds.load('dgs_corpus', builder_kwargs=dict(config=config))\n",
        "\n",
        "from sign_language_datasets.datasets.dgs_corpus.dgs_utils import get_elan_sentences\n",
        "\n",
        "for datum in itertools.islice(dgs_corpus[\"train\"], 0, 10):\n",
        "  elan_path = datum[\"paths\"][\"eaf\"].numpy().decode('utf-8')\n",
        "  sentences =  get_elan_sentences(elan_path)\n",
        "\n",
        "  try:\n",
        "    sentence = next(sentences)\n",
        "    print(\" \".join([s[\"gloss\"] for s in sentence[\"glosses\"]]))\n",
        "    print(sentence[\"german\"])\n",
        "    print()\n",
        "  except StopIteration:\n",
        "    pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Sentence level example (Videos are broken down to sentences)"
      ],
      "metadata": {
        "collapsed": false,
        "id": "6gaTQgmQX2Q3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "from sign_language_datasets.datasets.dgs_corpus import DgsCorpusConfig\n",
        "\n",
        "config = DgsCorpusConfig(name=\"only-annotations-sentence-level\", version=\"1.0.0\", include_video=False, include_pose=None, data_type=\"sentence\")\n",
        "dgs_corpus = tfds.load('dgs_corpus', builder_kwargs=dict(config=config))\n",
        "\n",
        "for datum in itertools.islice(dgs_corpus[\"train\"], 0, 5):\n",
        "  sentence = datum[\"sentence\"][\"german\"].numpy().decode('utf-8')\n",
        "  print(sentence)\n",
        "  print(datum)"
      ],
      "metadata": {
        "id": "hSWN-bpCX2Q3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# DGS Types"
      ],
      "metadata": {
        "collapsed": false,
        "id": "gyb2vSTLX2Q3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None, process_video=False)\n",
        "dgs_types = tfds.load('dgs_types', builder_kwargs=dict(config=config))\n",
        "\n",
        "for datum in itertools.islice(dgs_types[\"train\"], 0, 10):\n",
        "  print(datum)"
      ],
      "metadata": {
        "id": "RUbtkMMKX2Q3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sign Suisse"
      ],
      "metadata": {
        "collapsed": false,
        "id": "tS4GuMBbX2Q3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, process_video=False)\n",
        "sign_suisse = tfds.load('sign_suisse', builder_kwargs=dict(config=config))\n",
        "\n",
        "for datum in itertools.islice(sign_suisse[\"train\"], 0, 10):\n",
        "  print(datum)"
      ],
      "metadata": {
        "id": "kti4k3L3X2Q3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# NGT Corpus"
      ],
      "metadata": {
        "collapsed": false,
        "id": "KmJtA1UUX2Q3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "%%capture\n",
        "! pip install pympi-ling"
      ],
      "metadata": {
        "id": "vayqFcrKX2Q4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sign_language_datasets.datasets.ngt_corpus.ngt_corpus_utils import get_elan_sentences_ngt_corpus\n",
        "\n",
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n",
        "ngt = tfds.load(name='ngt_corpus', builder_kwargs={\"config\": config})\n",
        "\n",
        "for datum in itertools.islice(ngt[\"train\"], 0, 10):\n",
        "  print(datum['id'].numpy().decode('utf-8'))\n",
        "  elan_path = datum[\"paths\"][\"eaf\"].numpy().decode('utf-8')\n",
        "\n",
        "  sentences = get_elan_sentences_ngt_corpus(elan_path)\n",
        "\n",
        "  for sentence in sentences:\n",
        "    print(sentence)"
      ],
      "metadata": {
        "id": "TALQxYmeX2Q4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# BSL Corpus"
      ],
      "metadata": {
        "collapsed": false,
        "id": "Lc6eoG7OX2Q4"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "outputs": [],
      "source": [
        "from sign_language_datasets.datasets.bsl_corpus.bsl_corpus_utils import get_elan_sentences_bsl_corpus\n",
        "\n",
        "# this corpus requires a login\n",
        "\n",
        "BSLCP_USERNAME = \"\"\n",
        "BSLCP_PASSWORD = \"\"\n",
        "\n",
        "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None)\n",
        "\n",
        "bslcp = tfds.load(name='bsl_corpus', builder_kwargs={\"config\": config,\n",
        "                                                     \"bslcp_username\": BSLCP_USERNAME,\n",
        "                                                     \"bslcp_password\": BSLCP_PASSWORD})\n",
        "\n",
        "for datum in itertools.islice(bslcp[\"train\"], 0, 10):\n",
        "  print(datum['id'].numpy().decode('utf-8'))\n",
        "  elan_path = datum[\"paths\"][\"eaf\"][0].numpy().decode('utf-8')\n",
        "\n",
        "  sentences = get_elan_sentences_bsl_corpus(elan_path)\n",
        "\n",
        "  for sentence in sentences:\n",
        "    print(sentence)"
      ],
      "metadata": {
        "id": "UdrAR3ooX2Q4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# WMT-SLT\n",
        "\n",
        "Instructions and example code are here: https://github.com/sign-language-processing/datasets/blob/master/sign_language_datasets/datasets/wmt_slt/README.md"
      ],
      "metadata": {
        "collapsed": false,
        "id": "K984lKOuX2Q4"
      }
    }
  ]
}