Convert xmltodict result to structured data

[VietThan]

• cattrs version: 23.1.2
• Python version: 3.10.10
• Operating System: Mac

Description

Hi, I have XML files which I want to read and import into structured objects using attrs and cattrs. For xml reading, I'm using xmltodict, with the returned dictionary often having keys containing special characters like : or @. I've perused this issue python-attrs/attrs#417 and came up with the solution below but I'm getting an error which I don't understand.

What I Did

script:

# example.py
from __future__ import annotations

from typing import Any, List, Optional

from attrs import define, field
from cattrs import structure
from cattr.gen import make_dict_structure_fn, override
from cattr.preconf.json import make_converter

@define
class Ns2LogTransaction:
    _xmlns_ns2: str 
    """@xmlns:ns2"""

@define
class SBody:
    ns2_logTransaction: Ns2LogTransaction 
    """ns2:logTransaction"""

@define
class SEnvelope:
    _xmlns_S: str 
    """@xmlns:S"""
    S_Body: SBody 
    """S:Body"""

@define
class LogTransactionRequest:
    S_Envelope: SEnvelope
    """S:Envelope"""

    @classmethod
    def from_dict(cls, data: dict[str, Any]):
        """
        https://github.com/python-attrs/attrs/issues/417
        """

        converter = make_converter()
        converter.register_structure_hook(LogTransactionRequest, make_dict_structure_fn(LogTransactionRequest, converter, S_Envelope=override(rename='S:Envelope')))
        converter.register_structure_hook(SEnvelope, make_dict_structure_fn(SEnvelope, converter, _xmlns_S=override(rename='@xmlns:S')))
        converter.register_structure_hook(SEnvelope, make_dict_structure_fn(SEnvelope, converter, S_Body=override(rename='S:Body')))
        converter.register_structure_hook(SBody, make_dict_structure_fn(SBody, converter, ns2_logTransaction=override(rename='ns2:logTransaction')))
        converter.register_structure_hook(Ns2LogTransaction, make_dict_structure_fn(Ns2LogTransaction, converter, _xmlns_ns2=override(rename='@xmlns:ns2')))

        return converter.structure(data, cls)
    
if __name__ == "__main__":
    d = {'S:Envelope': {'@xmlns:S': 'http://schemas.xmlsoap.org/soap/envelope/', 'S:Body': {'ns2:logTransaction': {'@xmlns:ns2': 'http://random.url/1'}}}}
    lt = LogTransactionRequest.from_dict(d)

Traceback:

$ python example.py
/Users/vietthan/projects/jbrain_py/.venv/bin/python /Users/vietthan/projects/jbrain_py/example.py
  + Exception Group Traceback (most recent call last):
  |   File "/Users/vietthan/projects/jbrain_py/example.py", line 51, in <module>
  |     lt = LogTransactionRequest.from_dict(d)
  |   File "/Users/vietthan/projects/jbrain_py/example.py", line 47, in from_dict
  |     return converter.structure(data, cls)
  |   File "/Users/vietthan/projects/jbrain_py/.venv/lib/python3.10/site-packages/cattrs/converters.py", line 334, in structure
  |     return self._structure_func.dispatch(cl)(obj, cl)
  |   File "<cattrs generated structure __main__.LogTransactionRequest-2>", line 9, in structure_LogTransactionRequest
  |     if errors: raise __c_cve('While structuring ' + 'LogTransactionRequest', errors, __cl)
  | cattrs.errors.ClassValidationError: While structuring LogTransactionRequest (1 sub-exception)
  +-+---------------- 1 ----------------
    | Exception Group Traceback (most recent call last):
    |   File "<cattrs generated structure __main__.LogTransactionRequest-2>", line 5, in structure_LogTransactionRequest
    |     res['S_Envelope'] = __c_structure_S_Envelope(o['S:Envelope'], __c_type_S_Envelope)
    |   File "<cattrs generated structure __main__.SEnvelope-3>", line 14, in structure_SEnvelope
    |     if errors: raise __c_cve('While structuring ' + 'SEnvelope', errors, __cl)
    | cattrs.errors.ClassValidationError: While structuring SEnvelope (1 sub-exception)
    | Structuring class LogTransactionRequest @ attribute S_Envelope
    +-+---------------- 1 ----------------
      | Traceback (most recent call last):
      |   File "<cattrs generated structure __main__.SEnvelope-3>", line 5, in structure_SEnvelope
      |     res['xmlns_S'] = __c_structure__xmlns_S(o['_xmlns_S'])
      | KeyError: '_xmlns_S'
      | Structuring class SEnvelope @ attribute _xmlns_S
      +------------------------------------

[Tinche]

Hello,

you're on the right track! I've modified your example to this:

# example.py
from __future__ import annotations

from typing import Any

from attrs import define

from cattr.gen import make_dict_structure_fn, override
from cattr.preconf.json import make_converter


@define
class Ns2LogTransaction:
    _xmlns_ns2: str
    """@xmlns:ns2"""


@define
class SBody:
    ns2_logTransaction: Ns2LogTransaction
    """ns2:logTransaction"""


@define
class SEnvelope:
    _xmlns_S: str
    """@xmlns:S"""
    S_Body: SBody
    """S:Body"""


@define
class LogTransactionRequest:
    S_Envelope: SEnvelope
    """S:Envelope"""

    @classmethod
    def from_dict(cls, data: dict[str, Any]):
        """
        https://github.com/python-attrs/attrs/issues/417
        """

        converter = make_converter()
        converter.register_structure_hook(
            Ns2LogTransaction,
            make_dict_structure_fn(
                Ns2LogTransaction, converter, _xmlns_ns2=override(rename="@xmlns:ns2")
            ),
        )
        converter.register_structure_hook(
            SBody,
            make_dict_structure_fn(
                SBody,
                converter,
                ns2_logTransaction=override(rename="ns2:logTransaction"),
            ),
        )
        converter.register_structure_hook(
            SEnvelope,
            make_dict_structure_fn(
                SEnvelope,
                converter,
                _xmlns_S=override(rename="@xmlns:S"),
                S_Body=override(rename="S:Body"),
            ),
        )
        converter.register_structure_hook(
            LogTransactionRequest,
            make_dict_structure_fn(
                LogTransactionRequest,
                converter,
                S_Envelope=override(rename="S:Envelope"),
            ),
        )

        return converter.structure(data, cls)


if __name__ == "__main__":
    d = {
        "S:Envelope": {
            "@xmlns:S": "http://schemas.xmlsoap.org/soap/envelope/",
            "S:Body": {"ns2:logTransaction": {"@xmlns:ns2": "http://random.url/1"}},
        }
    }
    lt = LogTransactionRequest.from_dict(d)
    print(lt)

Notable changes:

• use only 1 call to generate the hook for SEnvelope, otherwise only the last generation will take effect
• change the order in which hooks are registered so that simpler hooks are registered first

One other thing: you shouldn't create a converter in the from_dict function, or if you do it should be cached, otherwise you'll get pretty bad performance.

Cattrs basically generates and compiles functions for un/structuring; this is why the order in which hooks are registered is important and why you want do avoid recreating converters.

Let me know if you have any other questions!

[VietThan]

Thank you! this is great! Will heed your advice!