Initial commit

.ci/aptPackagesToInstall.txt

@@ -0,0 +1,5 @@
+python3-lz4
+python3-msgpack
+python3-brotli
+python3-cbor
+python3-ujson

.ci/installDependencies.sh

@@ -0,0 +1,6 @@
+#/usr/bin/env sh
+
+apt-get update
+apt-get -y install python3-lz4 python3-msgpack python3-brotli python3-cbor python3-ujson
+pip3 install --upgrade https://gitlab.com/KOLANICH/py-lmdb/-/jobs/artifacts/gitlab/raw/wheels/lmdb-0.CI_cpython_latest-py3-none-any.whl?job=build
+python3 ./fix_python_modules_paths.py

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = true
+end_of_line = lf
+
+[*.{yml,yaml}]
+indent_style = space
+indent_size = 2

.github/.templateMarker

@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"

.github/workflows/CI.yml

@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,13 @@
+__pycache__
+*.pyc
+*.pyo
+*.egg-info
+build
+dist
+.eggs
+monkeytype.sqlite3
+/tests/**.sqlite
+/tests/databasesFiles
+/tests/**.mdb
+/tests/**.mdb-lock
+/.ipynb_checkpoints

.gitlab-ci.yml

@@ -0,0 +1,48 @@
+#image: pypy:latest
+image: registry.gitlab.com/kolanich-subgroups/docker-images/fixed_python:latest
+
+variables:
+  DOCKER_DRIVER: overlay2
+  SAST_ANALYZER_IMAGE_TAG: latest
+  SAST_DISABLE_DIND: "true"
+  SAST_CONFIDENCE_LEVEL: 5
+  CODECLIMATE_VERSION: latest
+
+include:
+  - template: SAST.gitlab-ci.yml
+  - template: Code-Quality.gitlab-ci.yml
+  - template: License-Management.gitlab-ci.yml
+
+build:
+  tags:
+    - shared
+    - linux
+  stage: build
+  variables:
+    GIT_DEPTH: "1"
+    PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
+
+  before_script:
+    - export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables`
+    - . ./.ci/installDependencies.sh
+
+  script:
+    - python3 setup.py bdist_wheel
+    - mv ./dist/*.whl ./dist/transformerz-0.CI-py3-none-any.whl
+    - pip3 install --upgrade -e ./[msgpack,lz4,brotli,zopflipy,cbor,zstd,lmdb]
+    - coverage run --source=transformerz -m pytest --junitxml=./rspec.xml ./tests/test.py
+    - coverage report -m
+    - coverage xml
+
+  coverage: /^TOTAL(?:\s+\d+){4}\s+(\d+%).+/
+
+  cache:
+    paths:
+      - $PYTHONUSERBASE
+
+  artifacts:
+    paths:
+      - dist
+    reports:
+      junit: ./rspec.xml
+      cobertura: ./coverage.xml

Code_Of_Conduct.md

@@ -0,0 +1 @@
+No codes of conduct!

MANIFEST.in

@@ -0,0 +1,4 @@
+include UNLICENSE
+include *.md
+include tests
+include .editorconfig

ReadMe.md

@@ -0,0 +1,50 @@
+transformerz.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/)
+===============
+~~[wheel (from GHA via `nightly.link`)](https://nightly.link/KOLANICH-libs/transformerz.py/workflows/CI/master/transformerz-0.CI-py3-none-any.whl)~~
+~~![GitLab Build Status](https://gitlab.com/KOLANICH/transformerz.py/badges/master/pipeline.svg)~~
+~~[![GitHub Actions](https://github.com/KOLANICH-libs/transformerz.py/workflows/CI/badge.svg)](https://github.com/KOLANICH-libs/transformerz.py/actions)~~
+~~![GitLab Coverage](https://gitlab.com/KOLANICH/transformerz.py/badges/master/coverage.svg)~~
+[![Libraries.io Status](https://img.shields.io/librariesio/github/KOLANICH-libs/transformerz.py.svg)](https://libraries.io/github/KOLANICH-libs/transformerz.py)
+[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://codeberg.org/KOLANICH-tools/antiflash.py)
+
+Just a set of composable processor objects that can be stacked, and path can be automatically routed.
+
+Each class/object has 2 members of type `type`:
+
+* `tgtType`
+* `srcType`
+
+and 2 functions
+
+* `process` - converts a value of `srcType` to the `tgtType`. Should parse the data from the representation useful for storing on disk.
+* `unprocess` - converts a value of  `tgtType` to `srcType` Should serialize the data from the representation useful for storing on disk.
+.
+
+The names of functions are inherited from `kaitai.process` library (Kaitai Struct is a parsing framework, so `process` historically means parsing), so are some classes (and I hope to get the stuff from this package merged supported by KS somewhen).
+
+There are 3 base classes:
+
+* `TransformerBase` - for objects with `srcType` and `tgtType` hardcoded in class definitions or available as props
+* `Transformer` - for objects with `srcType` and `tgtType` stored in slots
+* `FileTransformer` - for transformations when `unprocess`ed form can be stored in a file with a well-known extension and possible MIME type.
+* `BinaryProcessor` - adapter for Kaitai Struct stuff
+
+
+There are transformers are of different types and reside in different submodules:
+
+* `.serialization` - packages to serialize various objects
+	* `.json.jsonSerializer` -  Uses `ujson` if it is available which is faster than built-in `json` module.
+	* `.bson.bsonSerializer` - Available if `pymongo` is installed.
+	* `.msgpack.msgpackSerializer` - Available if a package for MsgPack serialization is installed.
+	* `.cbor.cborSerializer` - Available packages for CBOR serialization: either `cbor` or `cbor2` - are installed.
+	* `.pon.ponSerializer` - "Python Object Nonation" - stuff like JSON that can be safely evaluated using `literal_eval`
+* `.processors` - process binary data. This module contains the adapters allowing to use the stuff written to be used in `process` attr in Kaitai Struct specs.
+* `.compression` - packages to compress binary data. Take various params.
+* `.text` - convert text to bytes and back
+* `.struct` - parses data to tuples and back using `struct.Struct`. Numbers binary representations also go here. But not all. Some cannot be parsed by `struct`, so they go to ...
+* `.numpy` - parsing and serializing arrays of numbers using `numpy` machinery. Mostly needed for IEEE 751 floats not built into python.
+
+Tutorial
+--------
+
+The tutorial is available. [`tutorial.ipynb`](./tutorial.ipynb) [![NBViewer](https://nbviewer.org/static/ico/ipynb_icon_16x16.png)](https://nbviewer.org/urls/codeberg.org/KOLANICH-libs/transformerz.py/raw/master/tutorial.ipynb)

UNLICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org/>

fix_python_modules_paths.py

@@ -0,0 +1,54 @@
+import sys
+import os
+from pathlib import Path
+import sysconfig
+import re
+
+"""This tool makes available packages installed via apt to python installed in the original Docker image in /usr/local"""
+
+stdlibDir = Path(sysconfig.get_paths()["platstdlib"])
+
+cextNameRx = re.compile("^(?P<name>.+)\\.(?P<impl>cpython)-(?P<major>\\d)(?P<minor>\\d)m?-(?P<arch>x86_64)-(?P<abi>linux-gnu)\\.so$")
+
+
+def genNewName(name, impl, major, minor, arch, abi):
+	return name + "." + impl + "-" + str(major) + str(minor) + ("m" if (major, minor) < (3, 8) else "") + "-" + arch + "-" + abi + ".so"
+
+
+def symlink(f, to):
+	print(f, "->", to)
+	os.symlink(f, to)
+
+
+def genNewSoPath(parent, name):
+	m = cextNameRx.match(name)
+	if m:
+		d = m.groupdict()
+		d["major"] = sys.version_info[0]
+		d["minor"] = sys.version_info[1]
+		return parent / genNewName(**d)
+	else:
+		return parent / name
+
+
+def fixSoPath(f):
+	to = genNewSoPath(f.parent, f.name)
+	if not to.exists():
+		symlink(f, to)
+
+
+def fixPaths(fromAptPackagesInstalledPythonModulesDir):
+	for f in fromAptPackagesInstalledPythonModulesDir.glob("*"):
+		to = stdlibDir / f.name
+		if f.is_dir():
+			for l in f.glob("**/*.so"):
+				fixSoPath(l)
+		elif f.suffix == ".so":
+			to = genNewSoPath(stdlibDir, f.name)
+		if not to.exists():
+			symlink(f, to)
+
+
+if __name__ == "__main__":
+	print("stdlibDir:", stdlibDir)
+	fixPaths(Path("/usr/lib/python3/dist-packages/"))

pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = ["setuptools>=61.2.0", "wheel", "setuptools_scm[toml]>=3.4.3"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "transformerz"
+authors = [{name = "KOLANICH"}]
+description = "Just a library implementing composable and stackable processor objects for data."
+readme = "ReadMe.md"
+keywords = ["processors", "json", "cbor", "msgpack", "bson"]
+license = {text = "Unlicense"}
+classifiers = [
+	"Programming Language :: Python",
+	"Programming Language :: Python :: 3",
+	"Development Status :: 4 - Beta",
+	"Environment :: Other Environment",
+	"Intended Audience :: Developers",
+	"License :: Public Domain",
+	"Operating System :: OS Independent",
+	"Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.4"
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://codeberg.org/KOLANICH-libs/transformerz.py"
+
+[project.optional-dependencies]
+lz4 = ["lz4"]  # @ git+https://github.com/python-lz4/python-lz4
+brotli = ["brotli"]  # @ git+https://github.com/google/brotli
+zstd = ["zstandard"]  # @ git+https://github.com/indygreg/python-zstandard
+implode = [
+	"pkblast",  # @ git+https://codeberg.org/implode-compression-impls/pkblast.py
+	"pkimplode"  # @ git+https://codeberg.org/implode-compression-impls/pkimplode.py
+]
+lrzip = ["lrzip"]  # @ git+https://github.com/kata198/python-lrzip
+lzfse = ["pyliblzfse"]  # @ git+https://github.com/ydkhatri/pyliblzfse
+lzfu = ["compressed_rtf"]  # @ git+https://github.com/delimitry/compressed_rtf
+lzham = ["pylzham"]  # @ git+https://github.com/Galaxy1036/pylzham
+lzjb = ["lzjb"]  # @ git+https://github.com/unwind/python-lzjb
+lzo = ["lzo"]  # @ git+https://github.com/jd-boyd/python-lzo
+lzss = ["lzss"]  # @ git+https://github.com/yyogo/pylzss
+ppmd = ["pyppmd"]  # @ git+https://codeberg.org/miurahr/pyppmd
+snappy = ["python-snappy"]  # @ git+https://github.com/andrix/python-snappy
+
+bson = ["pymongo"] # @ git+https://github.com/mongodb/mongo-python-driver
+msgpack = ["msgpack"] # @ git+https://github.com/msgpack/msgpack-python
+yaml = ["ruamel.yaml"] # https://sourceforge.net/p/ruamel-yaml/code/ci/default/tarball
+libintl = ["polib"] # @ git+https://github.com/izimobil/polib/
+neon = ["neon-py"] # @ git+https://github.com/paveldedik/neon-py
+bencode = ["bencode"] 
+cbor = ["cbor2"] # @ git+https://github.com/agronholm/cbor2
+hcl2 = ["python-hcl2"] # @ git+https://github.com/amplify-education/python-hcl2
+rison = ["prison"] # @ git+https://github.com/betodealmeida/python-rison
+ubjson = ["ubjson"] # @ git+https://github.com/Iotic-Labs/py-ubjson
+
+[tool.setuptools]
+zip-safe = true
+include-package-data = false
+
+[tool.setuptools.packages]
+find = {namespaces = false, exclude=["libs", "tests"]}
+
+[tool.setuptools_scm]

tests/test.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+import typing
+import random
+import sqlite3
+import sys
+import unittest
+import warnings
+from functools import partial
+from pathlib import Path
+import random
+import sqlite3
+import unittest
+
+thisDir = Path(__file__).parent.absolute()
+sys.path.insert(0, str(thisDir.parent))
+
+
+def testString():
+	return "Aa12Бб_" + str(random.randint(1, 0xFFFFFFFF))  # nosec
+
+
+class SimpleTests(unittest.TestCase):
+	pass
+
+
+if __name__ == "__main__":
+	unittest.main()

transformerz/__init__.py

@@ -0,0 +1,22 @@
+import typing
+
+from .core import TransformerMirroringTgtType
+
+
+class DummyTransformer(TransformerMirroringTgtType):
+	"""Does nothing"""
+
+	__slots__ = ("tgtType",)
+
+	def __init__(self, name: str, tgtType: type) -> None:
+		self.tgtType = tgtType
+		super().__init__(name)
+
+	def unprocess(self, v: typing.Any) -> typing.Any:
+		return v
+
+	def process(self, v: typing.Any) -> typing.Any:
+		return v
+
+
+dummyTransformer = DummyTransformer("dummy", object)