diff --git a/README.md b/README.md index 780834f..f8d0ca7 100644 --- a/README.md +++ b/README.md @@ -42,25 +42,26 @@ rwth_phoenix2014_t = tfds.load(name='rwth_phoenix2014_t', builder_kwargs=dict(co ## Datasets -| Dataset | Videos | Poses | Versions | -|--------------------|--------------------------------------------------------------|--------------------------------------------------------|-----------| -| aslg_pc12 | N/A | N/A | 0.0.1 | -| rwth_phoenix2014_t | Yes | Holistic | 3.0.0 | -| autsl | Yes | OpenPose, Holistic | 1.0.0 | -| dgs_corpus | Yes | OpenPose, Holistic | 3.0.0 | -| dgs_types | Yes | | 3.0.0 | -| how2sign | Yes | OpenPose | 1.0.0 | -| sign2mint | Yes | | 1.0.0 | -| signtyp | Links | | 1.0.0 | -| swojs_glossario | Yes | | 1.0.0 | -| SignBank | N/A | | 1.0.0 | -| wlasl | [Failed](https://github.com/tensorflow/datasets/issues/2960) | [OpenPose](https://github.com/gulvarol/bsl1k/issues/4) | None | -| wmtslt | Yes | OpenPose, Holistic | 1.2.0 | -| msasl | | | None | -| Video-Based CSL | | | None | -| RVL-SLLL ASL | | | None | -| ngt_corpus | Yes | | 3.0.0 | -| bsl_corpus | No | No | 3.0.0 | +| Dataset | Videos | Poses | Versions | +|--------------------|--------------------------------------------------------------|--------------------------------------------------------|----------| +| aslg_pc12 | N/A | N/A | 0.0.1 | +| rwth_phoenix2014_t | Yes | Holistic | 3.0.0 | +| autsl | Yes | OpenPose, Holistic | 1.0.0 | +| dgs_corpus | Yes | OpenPose, Holistic | 3.0.0 | +| dgs_types | Yes | | 3.0.0 | +| how2sign | Yes | OpenPose | 1.0.0 | +| sign2mint | Yes | | 1.0.0 | +| signtyp | Links | | 1.0.0 | +| swojs_glossario | Yes | | 1.0.0 | +| SignBank | N/A | | 1.0.0 | +| wlasl | [Failed](https://github.com/tensorflow/datasets/issues/2960) | [OpenPose](https://github.com/gulvarol/bsl1k/issues/4) | None | +| wmtslt | Yes | OpenPose, Holistic | 1.2.0 | +| signsuisse | Yes | | 1.0.0 | +| msasl | | | None | +| Video-Based CSL | | | None | +| RVL-SLLL ASL | | | None | +| ngt_corpus | Yes | | 3.0.0 | +| bsl_corpus | No | No | 3.0.0 | ## Data Interface diff --git a/examples/load.ipynb b/examples/load.ipynb index aed2b9e..2fc9b31 100644 --- a/examples/load.ipynb +++ b/examples/load.ipynb @@ -20,10 +20,7 @@ "cell_type": "markdown", "metadata": { "id": "view-in-github", - "colab_type": "text", - "pycharm": { - "name": "#%% md\n" - } + "colab_type": "text" }, "source": [ "\"Open" @@ -32,10 +29,7 @@ { "cell_type": "code", "metadata": { - "id": "ov6fuFwGjlsy", - "pycharm": { - "name": "#%%\n" - } + "id": "ov6fuFwGjlsy" }, "source": [ "%%capture\n", @@ -47,10 +41,7 @@ { "cell_type": "code", "metadata": { - "id": "C4PZsi6pPp9j", - "pycharm": { - "name": "#%%\n" - } + "id": "C4PZsi6pPp9j" }, "source": [ "import tensorflow_datasets as tfds\n", @@ -65,10 +56,7 @@ { "cell_type": "markdown", "metadata": { - "id": "PKGZ4JXCZmSE", - "pycharm": { - "name": "#%% md\n" - } + "id": "PKGZ4JXCZmSE" }, "source": [ "# RWTH Phoenix 2014 T" @@ -77,10 +65,7 @@ { "cell_type": "code", "metadata": { - "id": "8wU1Q4URqRBE", - "pycharm": { - "name": "#%%\n" - } + "id": "8wU1Q4URqRBE" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"3.0.0\", include_video=False)\n", @@ -97,10 +82,7 @@ { "cell_type": "markdown", "metadata": { - "id": "v6iBwM9lTzS6", - "pycharm": { - "name": "#%% md\n" - } + "id": "v6iBwM9lTzS6" }, "source": [ "# Dicta Sign" @@ -109,10 +91,7 @@ { "cell_type": "code", "metadata": { - "id": "EQWUAgpVT0bK", - "pycharm": { - "name": "#%%\n" - } + "id": "EQWUAgpVT0bK" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None)\n", @@ -127,10 +106,7 @@ { "cell_type": "markdown", "metadata": { - "id": "OcIs13W6TfWz", - "pycharm": { - "name": "#%% md\n" - } + "id": "OcIs13W6TfWz" }, "source": [ "# ChicagoFSWild+" @@ -139,10 +115,7 @@ { "cell_type": "code", "metadata": { - "id": "o1X1kIgoTfec", - "pycharm": { - "name": "#%%\n" - } + "id": "o1X1kIgoTfec" }, "source": [ "# Version 2.0.0 is ChicagoFSWild+, 1.0.0 is ChicagoFSWild\n", @@ -158,10 +131,7 @@ { "cell_type": "markdown", "metadata": { - "id": "XK7jyOOtYv_P", - "pycharm": { - "name": "#%% md\n" - } + "id": "XK7jyOOtYv_P" }, "source": [ "# AUTSL" @@ -170,10 +140,7 @@ { "cell_type": "code", "metadata": { - "id": "dfZnI9K8YxfJ", - "pycharm": { - "name": "#%%\n" - } + "id": "dfZnI9K8YxfJ" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n", @@ -188,10 +155,7 @@ { "cell_type": "markdown", "metadata": { - "id": "rykmI68x3E07", - "pycharm": { - "name": "#%% md\n" - } + "id": "rykmI68x3E07" }, "source": [ "# SignBank" @@ -200,10 +164,7 @@ { "cell_type": "code", "metadata": { - "id": "12XcWfeg21kE", - "pycharm": { - "name": "#%%\n" - } + "id": "12XcWfeg21kE" }, "source": [ "signbank = tfds.load(name='sign_bank')\n", @@ -217,10 +178,7 @@ { "cell_type": "markdown", "metadata": { - "id": "biXjC80j17n1", - "pycharm": { - "name": "#%% md\n" - } + "id": "biXjC80j17n1" }, "source": [ "# SignTyp (https://signtyp.uconn.edu/signpuddle/index.php?ui=1&sgn=9032)\n" @@ -229,10 +187,7 @@ { "cell_type": "code", "metadata": { - "id": "dVgbyUIg165c", - "pycharm": { - "name": "#%%\n" - } + "id": "dVgbyUIg165c" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, extra={\"PHPSESSID\": \"hj9co07ct7f5noq529no9u09l4\"})\n", @@ -247,10 +202,7 @@ { "cell_type": "markdown", "metadata": { - "id": "yOLfw9-z2qK7", - "pycharm": { - "name": "#%% md\n" - } + "id": "yOLfw9-z2qK7" }, "source": [ "# Sign2Mint" @@ -259,10 +211,7 @@ { "cell_type": "code", "metadata": { - "id": "X96ogmu_22zv", - "pycharm": { - "name": "#%%\n" - } + "id": "X96ogmu_22zv" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n", @@ -277,10 +226,7 @@ { "cell_type": "markdown", "metadata": { - "id": "jnf4AaX936w4", - "pycharm": { - "name": "#%% md\n" - } + "id": "jnf4AaX936w4" }, "source": [ "# SWOJS Glossário" @@ -289,10 +235,7 @@ { "cell_type": "code", "metadata": { - "id": "shQxQtQP359y", - "pycharm": { - "name": "#%%\n" - } + "id": "shQxQtQP359y" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False)\n", @@ -310,10 +253,7 @@ { "cell_type": "markdown", "metadata": { - "id": "pNJdG7ExZugh", - "pycharm": { - "name": "#%% md\n" - } + "id": "pNJdG7ExZugh" }, "source": [ "# DGS Corpus" @@ -322,10 +262,7 @@ { "cell_type": "code", "metadata": { - "id": "TVjrhsbtbWbX", - "pycharm": { - "name": "#%%\n" - } + "id": "TVjrhsbtbWbX" }, "source": [ "%%capture\n", @@ -340,19 +277,13 @@ "## Document Level example (Long videos)" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false } }, { "cell_type": "code", "metadata": { - "id": "m1XvMTs9Zvx4", - "pycharm": { - "name": "#%%\n" - } + "id": "m1XvMTs9Zvx4" }, "source": [ "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, include_pose=None)\n", @@ -381,10 +312,7 @@ "## Sentence level example (Videos are broken down to sentences)" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false } }, { @@ -403,10 +331,7 @@ " print(datum)" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } + "collapsed": false } }, { @@ -415,10 +340,7 @@ "# DGS Types" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false } }, { @@ -433,10 +355,31 @@ " print(datum)" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } + "collapsed": false + } + }, + { + "cell_type": "markdown", + "source": [ + "# Sign Suisse" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "config = SignDatasetConfig(name=\"only-annotations\", version=\"1.0.0\", include_video=False, process_video=False)\n", + "sign_suisse = tfds.load('sign_suisse', builder_kwargs=dict(config=config))\n", + "\n", + "for datum in itertools.islice(sign_suisse[\"train\"], 0, 10):\n", + " print(datum)" + ], + "metadata": { + "collapsed": false } }, { @@ -445,10 +388,7 @@ "# NGT Corpus" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false } }, { @@ -460,10 +400,7 @@ "! pip install pympi-ling" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } + "collapsed": false } }, { @@ -484,10 +421,7 @@ " print(sentence)" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } + "collapsed": false }, "execution_count": null, "outputs": [] @@ -498,10 +432,7 @@ "# BSL Corpus" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false } }, { @@ -532,10 +463,7 @@ " print(sentence)" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%%\n" - } + "collapsed": false } }, { @@ -546,11 +474,8 @@ "Instructions and example code are here: https://github.com/sign-language-processing/datasets/blob/master/sign_language_datasets/datasets/wmt_slt/README.md" ], "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } + "collapsed": false } } ] -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index d4b9307..459801d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ Pillow opencv-python==4.5.5.64 requests etils +lxml \ No newline at end of file diff --git a/setup.py b/setup.py index ee33585..0fbe3e7 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name="sign-language-datasets", packages=packages, - version="0.1.5", + version="0.1.6", description="TFDS Datasets for sign language", author="Amit Moryossef", author_email="amitmoryossef@gmail.com", diff --git a/sign_language_datasets/datasets/__init__.py b/sign_language_datasets/datasets/__init__.py index 435164b..ee359a4 100644 --- a/sign_language_datasets/datasets/__init__.py +++ b/sign_language_datasets/datasets/__init__.py @@ -10,6 +10,7 @@ from .sign2mint import Sign2MINT from .signbank import SignBank from .signtyp import SignTyp +from .signsuisse import SignSuisse from .swojs_glossario import SwojsGlossario from .wlasl import Wlasl from .ngt_corpus import NGTCorpus diff --git a/sign_language_datasets/datasets/signsuisse/__init__.py b/sign_language_datasets/datasets/signsuisse/__init__.py new file mode 100644 index 0000000..08605ce --- /dev/null +++ b/sign_language_datasets/datasets/signsuisse/__init__.py @@ -0,0 +1,3 @@ +"""signsuisse dataset.""" + +from .signsuisse import SignSuisse diff --git a/sign_language_datasets/datasets/signsuisse/checksums.tsv b/sign_language_datasets/datasets/signsuisse/checksums.tsv new file mode 100644 index 0000000..e69de29 diff --git a/sign_language_datasets/datasets/signsuisse/dummy_data/TODO-add_fake_data_in_this_directory.txt b/sign_language_datasets/datasets/signsuisse/dummy_data/TODO-add_fake_data_in_this_directory.txt new file mode 100644 index 0000000..e69de29 diff --git a/sign_language_datasets/datasets/signsuisse/signsuisse.py b/sign_language_datasets/datasets/signsuisse/signsuisse.py new file mode 100644 index 0000000..d20d072 --- /dev/null +++ b/sign_language_datasets/datasets/signsuisse/signsuisse.py @@ -0,0 +1,167 @@ +"""A Swiss Sign Language Lexicon, combines all three Swiss sign languages: the German-Swiss Sign Language (DSGS), the Langue des Signes Française (LSF) and the Lingua Italiana dei Segni (LIS). .""" +import json +import re +import string + +import tensorflow_datasets as tfds + +from ..warning import dataset_warning +from ...datasets import SignDatasetConfig + +_DESCRIPTION = """ +Ein umfangreiches interaktives Lexikon der drei Gebärdensprachen der Schweiz (Deutschschweizerische Gebärdensprache DSGS, Langue des Signes Française LSF und Lingua Italiana dei Segni LIS). Herausgegeben vom Schweizerischen Gehörlosenbund (SGB-FSS).""" + +# TODO(signsuisse): BibTeX citation +_CITATION = """ +""" + +SITE_URL = "https://signsuisse.sgb-fss.ch" + + +class SignSuisse(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for signsuisse dataset.""" + + VERSION = tfds.core.Version("1.0.0") + RELEASE_NOTES = { + "1.0.0": "Initial crawl.", + } + + BUILDER_CONFIGS = [ + SignDatasetConfig(name="default", include_video=True), + SignDatasetConfig(name="annotations", include_video=False), + ] + + def _info(self) -> tfds.core.DatasetInfo: + """Return s the dataset metadata.""" + + features = { + "id": tfds.features.Text(), + "name": tfds.features.Text(), + "category": tfds.features.Text(), + "spokenLanguage": tfds.features.Text(), + "signedLanguage": tfds.features.Text(), + "url": tfds.features.Text(), + "paraphrase": tfds.features.Text(), + "definition": tfds.features.Text(), + "exampleText": tfds.features.Text(), + "exampleVideo": tfds.features.Text(), + } + + if self._builder_config.include_video and self._builder_config.process_video: + features["video"] = self._builder_config.video_feature((640, 480)) + else: + features["video"] = tfds.features.Text() + + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict(features), + homepage=SITE_URL, + supervised_keys=None, + citation=_CITATION, + ) + + def _list_all_lexicon_items(self, dl_manager: tfds.download.DownloadManager): + # The lexicon does not allow free search. One must search at least two letters. + letters = [c1 + c2 for c1 in string.ascii_lowercase for c2 in string.ascii_lowercase] + search_url = SITE_URL + "/index.php?eID=signsuisse_search&sword=" + urls = [search_url + l for l in letters] + indexes = dl_manager.download(urls) + + lexicon_items = {} + for index in indexes: + with open(index, "r", encoding="utf-8") as f: + index = json.load(f) + for item in index["items"]: + lexicon_items[item["uid"]] = item + + return lexicon_items.values() + + def _parse_item(self, item, item_page): + item["name"] = item["name"].replace(" ", " ").replace(" ", " ") + with open(item_page, "r", encoding="utf-8") as f: + html = f.read() + + # Verify that the page matches the item + title = re.search(r"(.*?)", html).group(1).strip() + title = title.replace("&", "&") + if title != item["name"]: + raise ValueError("Title does not match item name") + + assert title == item["name"] + + example = re.search(r"Beispiel

(.*?)

.*?

(.*?)

", html) + paraphrase = paraphrase_match.group(1).strip() if paraphrase_match else "" + definition_match = re.search(r"Definition

(.*?)

", html) + definition = definition_match.group(1).strip() if definition_match else "" + + return { + "id": item["uid"], + "name": item["name"], + "category": item["kategorie"], + "spokenLanguage": item["sprache"], + "signedLanguage": "ch-" + item["sprache"], + "url": SITE_URL + item["link"], + "paraphrase": paraphrase, + "definition": definition, + "exampleText": example_text, + "exampleVideo": example_video, + "video": video + } + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + dataset_warning(self) + print( + "The lexicon is available free of charge, so we look forward to your donation! https://www.sgb-fss.ch/spenden/jetzt-spenden/") + + lexicon_items = self._list_all_lexicon_items(dl_manager) + print("Found", len(lexicon_items), "lexicon items.") + item_urls = [SITE_URL + item["link"] for item in lexicon_items] + # Item URLS are actually too long. We need to shorten them. + item_urls = [url[:url.find("&tx_issignsuisselexikon_anzeige%5Baction")] for url in item_urls] + items_pages = dl_manager.download(item_urls) + + data = [] + for item, item_page in zip(lexicon_items, items_pages): + try: + item = self._parse_item(item, item_page) + if item is not None: + data.append(item) + except Exception as e: + print("Failed to parse item") + print(item) + print(item_page) + print(e) + print("\n\n\n\n\n\n") + # raise e + + # Download videos if requested + if self._builder_config.include_video: + video_urls = [item["video"] for item in data] + videos = dl_manager.download(video_urls) + for datum, video in zip(data, videos): + datum["video"] = video + if not self._builder_config.process_video: + datum["video"] = str(datum["video"]) + + + return {"train": self._generate_examples(data)} + + def _generate_examples(self, data): + for datum in data: + yield datum["id"], datum diff --git a/sign_language_datasets/datasets/signsuisse/signsuisse_test.py b/sign_language_datasets/datasets/signsuisse/signsuisse_test.py new file mode 100644 index 0000000..c2f4294 --- /dev/null +++ b/sign_language_datasets/datasets/signsuisse/signsuisse_test.py @@ -0,0 +1,25 @@ +"""signsuisse dataset.""" + +import tensorflow_datasets as tfds +from . import signsuisse + + +class SignSuisseTest(tfds.testing.DatasetBuilderTestCase): + """Tests for signsuisse dataset.""" + + # TODO(signsuisse): + DATASET_CLASS = signsuisse.SignSuisse + SPLITS = { + "train": 3, # Number of fake train example + "test": 1, # Number of fake test example + } + + # If you are calling `download/download_and_extract` with a dict, like: + # dl_manager.download({'some_key': 'http://a.org/out.txt', ...}) + # then the tests needs to provide the fake output paths relative to the + # fake data directory + # DL_EXTRACT_RESULT = {'some_key': 'output_file1.txt', ...} + + +if __name__ == "__main__": + tfds.testing.test_main() diff --git a/sign_language_datasets/utils/signwriting/ocr/ocr.py b/sign_language_datasets/utils/signwriting/ocr/ocr.py index f46eaea..c63f589 100644 --- a/sign_language_datasets/utils/signwriting/ocr/ocr.py +++ b/sign_language_datasets/utils/signwriting/ocr/ocr.py @@ -8,7 +8,6 @@ import numpy as np from numpy.lib.stride_tricks import as_strided import cv2 -from PIL import Image, ImageDraw, ImageFont import os @@ -57,6 +56,8 @@ def crop_whitespace(img): @functools.lru_cache() def get_font(): + from PIL import ImageFont + dirname = os.path.dirname(__file__) font_path = os.path.join(dirname, "assets/SuttonSignWritingOneD.ttf") @@ -64,6 +65,12 @@ def get_font(): def image_to_fsw(image: np.ndarray, symbols: List[str]) -> str: + try: + from PIL import Image, ImageDraw + except ImportError: + raise ImportError("Please install pillow with: pip install Pillow") + + font = get_font() # Adding border for conv calc to go over borders