add HashName

allennlp_dataframe_mapper/transforms/__init__.py

@@ -1,4 +1,5 @@
 from allennlp_dataframe_mapper.transforms.base import RegistrableTransform  # NOQA
+from allennlp_dataframe_mapper.transforms.hash_name import HashName  # NOQA
 from allennlp_dataframe_mapper.transforms.preprocessing import (  # NOQA
     FlattenTransformer,
     LabelEncoder,

allennlp_dataframe_mapper/transforms/hash_name.py

@@ -0,0 +1,27 @@
+import hashlib
+from typing import Optional
+
+import numpy as np
+from allennlp_dataframe_mapper.transforms import RegistrableTransform
+from overrides import overrides
+
+
+@RegistrableTransform.register("hashname")
+class HashName(RegistrableTransform):
+    def __init__(self, ext: Optional[str] = None) -> None:
+        super().__init__()
+        self._ext = ext
+
+    def to_hash_name(self, url: str) -> str:
+        hash_name = hashlib.md5(url.encode("utf8")).hexdigest()
+        if self._ext is not None:
+            return hash_name + self._ext
+        return hash_name
+
+    @overrides
+    def fit(self, *args, **kwargs):
+        return self
+
+    @overrides
+    def transform(self, X):
+        return np.vectorize(self.to_hash_name)(X)

test_fixtures/transforms/hash_name/hash_name.jsonnet

@@ -0,0 +1,3 @@
+{
+    "type": "hashname"
+}

tests/transforms/base_test.py

@@ -15,6 +15,7 @@ def test_registrable_transform(self) -> None:
                 "label-encoder",
                 "logarithmer",
                 "flatten",
+                "hashname",
             ]
         )
 
@@ -25,3 +26,4 @@ def test_registrable_transform(self) -> None:
         assert RegistrableTransform.by_name("label-encoder").__name__ == "LabelEncoder"
         assert RegistrableTransform.by_name("logarithmer").__name__ == "Logarithmer"
         assert RegistrableTransform.by_name("flatten").__name__ == "FlattenTransformer"
+        assert RegistrableTransform.by_name("hashname").__name__ == "HashName"

tests/transforms/hash_name_test.py

@@ -0,0 +1,60 @@
+import numpy as np
+import pytest
+from allennlp.common import Params
+from allennlp_dataframe_mapper.common.testing import AllenNlpDataFrameMapperTestCase
+from allennlp_dataframe_mapper.transforms import HashName, RegistrableTransform
+
+
+class TestHashName(AllenNlpDataFrameMapperTestCase):
+    def test_hash_name_from_params(self):
+        params = Params.from_file(
+            self.FIXTURES_ROOT / "transforms" / "hash_name" / "hash_name.jsonnet"
+        )
+        transform = RegistrableTransform.from_params(params=params)
+        assert isinstance(transform, HashName)
+
+    @pytest.mark.parametrize(
+        "url, expected",
+        (
+            ("http://example.com/", "a6bf1757fff057f266b697df9cf176fd"),
+            ("http://abehiroshi.la.coocan.jp/", "99daa109463db188c4160ee06a6ca8f7"),
+        ),
+    )
+    def test_hash_name(self, url, expected):
+        hash_name = HashName()
+        url_hashed = hash_name.transform(url)
+
+        expected_np = np.array(expected)
+        assert np.array_equal(url_hashed, expected_np)
+
+    def test_hash_names(self):
+
+        urls = ["http://example.com/", "http://abehiroshi.la.coocan.jp/"]
+        hash_name = HashName()
+        url_hashed = hash_name.transform(urls)
+
+        expected = np.array(
+            [
+                "a6bf1757fff057f266b697df9cf176fd",
+                "99daa109463db188c4160ee06a6ca8f7",
+            ]
+        )
+        assert np.array_equal(url_hashed, expected)
+
+    @pytest.mark.parametrize(
+        "ext, url, expected",
+        (
+            (".png", "http://example.com/", "a6bf1757fff057f266b697df9cf176fd"),
+            (
+                ".jpg",
+                "http://abehiroshi.la.coocan.jp/",
+                "99daa109463db188c4160ee06a6ca8f7",
+            ),
+        ),
+    )
+    def test_hash_name_with_ext(self, ext, url, expected):
+        hash_name = HashName(ext=ext)
+        url_hashed = hash_name.transform(url)
+
+        expected_np = np.array(expected + ext)
+        assert np.array_equal(url_hashed, expected_np)