🗓 Aug 10, 2022 12:07:39 AM

🎨 from/to base64 simplified and handles urlsafe natively
🚧 swap_endianness
🔍 user supplied format string for from_unix_ts
🔥 unix_ts renamed to unix_timestamp
🧪 tests added/updated

TODO

@@ -14,7 +14,8 @@ New ideas:
   ☐ ignore error method
   ☐ swap bytes
   ☐ @high disable plugins from code
-  homophonic decoder
+  ☐ homophonic decoder
+  ☐ append method for core to add data to the state
 
 Bug:
 

chepy/core.py

@@ -940,7 +940,7 @@ def run_recipe(self, recipes: List[Mapping[str, Union[str, Mapping[str, Any]]]])
             Chepy: The Chepy object.
 
         Examples:
-            >>> c = Chepy('bG9sCg==').run_recipe([{"function":"base64_decode","args":{"custom":None,"fix_padding":True}}]])
+            >>> c = Chepy('bG9sCg==').run_recipe([{"function":"base64_decode","args":{"custom":None}}]])
             >>> lol
             In this example, we are calling the base64 decode method on the state.
         """

chepy/modules/dataformat.py

@@ -22,8 +22,6 @@
 DataFormatT = TypeVar("DataFormatT", bound="DataFormat")
 
 
-
-
 class DataFormat(ChepyCore):
     def __init__(self, *data):
         super().__init__(*data)
@@ -309,7 +307,7 @@ def base32_decode(self) -> DataFormatT:
         return self
 
     @ChepyDecorators.call_stack
-    def base91_encode(self) -> DataFormatT: # pragma: no cover
+    def base91_encode(self) -> DataFormatT:  # pragma: no cover
         """Base91 encode
         Reference: https://github.com/aberaud/base91-python/blob/master/base91.py#L69
 
@@ -333,7 +331,9 @@ def base91_encode(self) -> DataFormatT: # pragma: no cover
                     v = b & 16383
                     b >>= 14
                     n -= 14
-                out += Encoding.BASE91_ALPHABET[v % 91] + Encoding.BASE91_ALPHABET[v // 91]
+                out += (
+                    Encoding.BASE91_ALPHABET[v % 91] + Encoding.BASE91_ALPHABET[v // 91]
+                )
         if n:
             out += Encoding.BASE91_ALPHABET[b % 91]
             if n > 7 or b > 90:
@@ -342,7 +342,7 @@ def base91_encode(self) -> DataFormatT: # pragma: no cover
         return self
 
     @ChepyDecorators.call_stack
-    def base91_decode(self) -> DataFormatT: # pragma: no cover
+    def base91_decode(self) -> DataFormatT:  # pragma: no cover
         """Decode as Base91
         Reference: https://github.com/aberaud/base91-python/blob/master/base91.py#L42
 
@@ -416,7 +416,7 @@ def from_bytes(self) -> DataFormatT:
         return self
 
     @ChepyDecorators.call_stack
-    def base64_encode(self, custom: str = None) -> DataFormatT:
+    def base64_encode(self, custom: str = None, url_safe: bool = False) -> DataFormatT:
         """Encode as Base64
 
         Base64 is a notation for encoding arbitrary byte data using a
@@ -426,6 +426,7 @@ def base64_encode(self, custom: str = None) -> DataFormatT:
 
         Args:
             custom (str, optional): Provide a custom charset to base64 with
+            url_safe (bool, optional): Encode with url safe charset.
 
         Returns:
             Chepy: The Chepy object.
@@ -436,6 +437,9 @@ def base64_encode(self, custom: str = None) -> DataFormatT:
             >>> Chepy("Some data").base64_encode(custom=custom).o
             b'IqxhNG/YMLFV'
         """
+        if url_safe:
+            self.state = base64.urlsafe_b64encode(self._convert_to_bytes())
+            return self
         if custom is not None:
             x = base64.b64encode(self._convert_to_bytes())
             std_base64chars = (
@@ -450,9 +454,7 @@ def base64_encode(self, custom: str = None) -> DataFormatT:
         return self
 
     @ChepyDecorators.call_stack
-    def base64_decode(
-        self, custom: str = None, fix_padding: bool = True
-    ) -> DataFormatT:
+    def base64_decode(self, custom: str = None, url_safe: bool = False) -> DataFormatT:
         """Decode as Base64
 
         Base64 is a notation for encoding arbitrary byte data using a
@@ -462,7 +464,7 @@ def base64_decode(
 
         Args:
             custom (str, optional): Provide a custom charset to base64 with
-            fix_padding (bool, optional): If padding error, add padding. Defaults to True
+            url_safe (bool, optional): If true, decode url safe. Defaults to False
 
         Returns:
             Chepy: The Chepy object.
@@ -474,25 +476,17 @@ def base64_decode(
             >>> c.out()
             b"some random? data"
         """
+        data = self._convert_to_str()
         if custom is not None:
             std_base64chars = (
                 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
             )
-            c = self._convert_to_str().translate(str.maketrans(custom, std_base64chars))
-            self.state = base64.b64decode(c.encode())
+            data = data.translate(str.maketrans(custom, std_base64chars))
+        data += "=="
+        if url_safe:
+            self.state = base64.urlsafe_b64decode(data)
         else:
-            try:
-                self.state = base64.b64decode(self._convert_to_bytes())
-            except binascii.Error:
-                if fix_padding:
-                    try:
-                        self._warning_logger("Padding error. Adding =")
-                        self.state = base64.b64decode(self._convert_to_bytes() + b"=")
-                    except binascii.Error:  # pragma: no cover
-                        self._warning_logger("Padding error. Adding ==")
-                        self.state = base64.b64decode(self._convert_to_bytes() + b"==")
-                else:  # pragma: no cover
-                    raise
+            self.state = base64.b64decode(data)
         return self
 
     @ChepyDecorators.call_stack
@@ -1299,3 +1293,18 @@ def remove_nonprintable(self, replace_with: bytes = b"") -> DataFormatT:
         data = self._convert_to_bytes()
         self.state = re.sub(b"[^[:print:]]", replace_with, data)
         return self
+
+    @ChepyDecorators.call_stack
+    def swap_endianness(self) -> DataFormatT:
+        # TODO make this better. this is not inline with cyberchef
+        """Swap endianness of a hex string.
+
+        Returns:
+            Chepy: The Chepy object.
+        """
+        data = self._convert_to_bytes()
+        # check if hex
+        if not re.match(b"^[0-9a-fA-F]+$", data): # pragma: no cover
+            raise ValueError("Data is not hex")
+        self.state = hex(struct.unpack("<I", struct.pack(">I", int(data, 16)))[0])[2:]
+        return self

chepy/modules/dataformat.pyi

@@ -29,7 +29,7 @@ class DataFormat(ChepyCore):
     def to_bytes(self: DataFormatT) -> DataFormatT: ...
     def from_bytes(self: DataFormatT) -> DataFormatT: ...
     def base64_encode(self: DataFormatT, custom: str=...) -> DataFormatT: ...
-    def base64_decode(self: DataFormatT, custom: str=..., fix_padding: bool=...) -> DataFormatT: ...
+    def base64_decode(self: DataFormatT, custom: str=..., url_safe: bool=...) -> DataFormatT: ...
     def decode_bytes(self: DataFormatT, errors: Literal['ignore', 'backslashreplace', 'replace']=...) -> DataFormatT: ...
     def to_hex(self: DataFormatT, delimiter: str=..., join_by: str=...) -> DataFormatT: ...
     def from_hex(self: DataFormatT, delimiter: str=...) -> DataFormatT: ...
@@ -76,3 +76,4 @@ class DataFormat(ChepyCore):
     def remove_nonprintable(self: DataFormatT, replace_with: bytes = ...): ...
     def base91_encode(self: DataFormatT) -> DataFormatT: ...
     def base91_decode(self: DataFormatT) -> DataFormatT: ...
+    def swap_endianness(self: DataFormatT) -> DataFormatT: ...

chepy/modules/datetimemodule.py

@@ -12,21 +12,24 @@ def __init__(self, *data):
         super().__init__(*data)
 
     @ChepyDecorators.call_stack
-    def from_unix_ts(self) -> DateTimeT:
+    def from_unix_timestamp(self, format: str='%c') -> DateTimeT:
         """Convert UNIX timestamp to datetime
 
+        Args:
+            format: Format to use for datetime.strftime()
+
         Returns:
             Chepy: The Chepy object.
 
         Examples:
-            >>> Chepy("1573426649").from_unix_ts()
+            >>> Chepy("1573426649").from_unix_timestamp()
             "Sun Nov 10 17:57:29 2019"
         """
-        self.state = datetime.fromtimestamp(self._convert_to_int()).strftime("%c")
+        self.state = datetime.fromtimestamp(self._convert_to_int()).strftime(format)
         return self
 
     @ChepyDecorators.call_stack
-    def to_unix_ts(self) -> DateTimeT:  # pragma: no cover
+    def to_unix_timestamp(self) -> DateTimeT:  # pragma: no cover
         """Convert datetime string to unix ts
 
         The format for the string is %a %b %d %H:%M:%S %Y, which is equivalent to
@@ -36,7 +39,7 @@ def to_unix_ts(self) -> DateTimeT:  # pragma: no cover
             Chepy: The Chepy object.
 
         Examples:
-            >>> Chepy("Sun Nov 10 17:57:29 2019").to_unix_ts()
+            >>> Chepy("Sun Nov 10 17:57:29 2019").to_unix_timestamp()
             "1573426649"
         """
         self.state = int(

chepy/modules/datetimemodule.pyi

@@ -1,10 +1,10 @@
 from ..core import ChepyCore
 from typing import Any, TypeVar
 
-DateTimeT = TypeVar('DateTimeT', bound='DateTime')
+DateTimeT = TypeVar("DateTimeT", bound="DateTime")
 
 class DateTime(ChepyCore):
     def __init__(self, *data: Any) -> None: ...
     state: Any = ...
-    def from_unix_ts(self: DateTimeT) -> DateTimeT: ...
-    def to_unix_ts(self: DateTimeT) -> DateTimeT: ...
+    def from_unix_timestamp(self: DateTimeT, format: str = ...) -> DateTimeT: ...
+    def to_unix_timestamp(self: DateTimeT) -> DateTimeT: ...

chepy/modules/utils.py

@@ -123,7 +123,7 @@ def regex_search(
         unicode: bool = False,
         extended: bool = False,
     ) -> UtilsT:
-        """Regex search on current data
+        """Regex search on current data. State will be an array of matches.
 
         Args:
             pattern (str): Required. The regex pattern to search by

tests/test_core.py

@@ -148,7 +148,7 @@ def test_run_recipe():
             recipes=[
                 {
                     "function": "base64_decode",
-                    "args": {"custom": None, "fix_padding": True},
+                    "args": {"custom": None},
                 },
                 {"function": "swap_case", "args": {}},
             ]

tests/test_dataformat.py

@@ -112,6 +112,7 @@ def test_base64_encode():
         .o
         == b"QqxhNG/mMKtYPqoz64FVR42="
     )
+    assert Chepy("test").base64_encode(url_safe=True).o == b"dGVzdA=="
 
 
 def test_base64_decode():
@@ -124,6 +125,7 @@ def test_base64_decode():
         .o
         == b"some random? data"
     )
+    assert Chepy("dGVzdA").base64_decode(url_safe=True).o == b"test"
 
 
 def test_decode_bytes():
@@ -401,3 +403,7 @@ def test_base91():
     out = "@iH<,{_{W$OsuxXi%]D"
     assert Chepy(data).base91_encode().o == out
     assert Chepy(out).base91_decode().o.decode() == data
+
+
+def test_swap_endianness():
+    assert Chepy("01020304").swap_endianness().o == "4030201"

tests/test_datetime.py

@@ -1,9 +1,9 @@
 from chepy import Chepy
 
 
-def test_from_unix_ts():
-    assert Chepy("1573426649").from_unix_ts().o[-4:] == "2019"
+def test_from_unix_timestamp():
+    assert Chepy("1573426649").from_unix_timestamp().o[-4:] == "2019"
 
 
-# def test_to_unix_ts():
-#     assert Chepy("Sun Nov 10 17:57:29 2019").to_unix_ts().o == 1573426649
+# def test_to_unix_timestamp():
+#     assert Chepy("Sun Nov 10 17:57:29 2019").to_unix_timestamp().o == 1573426649

tests/test_links.py

@@ -21,5 +21,5 @@ def test_google_search_ei_to_epoch():
     assert Chepy("Bh8hYqykHc64mAXkkoTgCg==").google_search_ei_to_epoch().o == 1646337798
 
 
-# def test_to_unix_ts():
-#     assert Chepy("Sun Nov 10 17:57:29 2019").to_unix_ts().o == 1573426649
+# def test_to_unix_timestamp():
+#     assert Chepy("Sun Nov 10 17:57:29 2019").to_unix_timestamp().o == 1573426649