diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3f1bcccd..fb6dc1e4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.33.0 +current_version = 0.34.0 commit = True tag = False message = chore: Bump version from {current_version} to {new_version} diff --git a/HISTORY.md b/HISTORY.md index e25c031b..8a8198b2 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,13 @@ # History +## 0.34.0 (2024-09-26) + +- (PR #690, 2024-09-25) chore(deps): Bump lxml from 5.2.2 to 5.3.0 +- (PR #691, 2024-09-25) chore(deps): Bump marshmallow from 3.21.3 to 3.22.0 +- (PR #701, 2024-09-25) Enable type checking for `setuptools` +- (PR #702, 2024-09-25) Enable type checking for `lxml` +- (PR #703, 2024-09-26) Relax some validations for trusted inputs + ## 0.33.0 (2024-09-24) - (PR #689, 2024-09-24) chore(deps): Bump pydantic from 2.7.2 to 2.8.2 diff --git a/mypy.ini b/mypy.ini index 4a64dbc1..6cadfb87 100644 --- a/mypy.ini +++ b/mypy.ini @@ -32,15 +32,9 @@ ignore_missing_imports = True [mypy-django_filters.*] ignore_missing_imports = True -[mypy-lxml.*] -ignore_missing_imports = True - [mypy-rest_framework.*] ignore_missing_imports = True -[mypy-setuptools.*] -ignore_missing_imports = True - [pydantic-mypy] init_forbid_extra = True init_typed = True diff --git a/requirements-dev.in b/requirements-dev.in index 1805d67e..9b49d678 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -15,6 +15,7 @@ pip-tools==7.4.1 tox==4.20.0 twine==5.1.1 types-jsonschema==4.23.0.20240813 +types-lxml==2024.9.16 types-pyOpenSSL==24.1.0.20240722 types-pytz==2024.2.0.20240913 wheel==0.44.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f39f580..61de4934 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -45,6 +45,8 @@ cryptography==43.0.1 # -c requirements.txt # secretstorage # types-pyopenssl +cssselect==1.2.0 + # via types-lxml distlib==0.3.7 # via virtualenv docutils==0.19 @@ -157,10 +159,16 @@ tox==4.20.0 # via -r requirements-dev.in twine==5.1.1 # via -r requirements-dev.in +types-beautifulsoup4==4.12.0.20240907 + # via types-lxml types-cffi==1.16.0.20240331 # via types-pyopenssl +types-html5lib==1.1.11.20240806 + # via types-beautifulsoup4 types-jsonschema==4.23.0.20240813 # via -r requirements-dev.in +types-lxml==2024.9.16 + # via -r requirements-dev.in types-pyopenssl==24.1.0.20240722 # via -r requirements-dev.in types-pytz==2024.2.0.20240913 @@ -173,6 +181,7 @@ typing-extensions==4.12.2 # black # mypy # rich + # types-lxml urllib3==1.26.19 # via # requests diff --git a/requirements.in b/requirements.in index 664524fd..85236d9d 100644 --- a/requirements.in +++ b/requirements.in @@ -13,8 +13,8 @@ Django>=2.2.24 djangorestframework>=3.10.3,<3.16 importlib-metadata==8.4.0 jsonschema==4.23.0 -lxml==5.2.2 -marshmallow==3.21.3 +lxml==5.3.0 +marshmallow==3.22.0 pydantic==2.9.2 pyOpenSSL==24.2.1 pytz==2024.1 diff --git a/requirements.txt b/requirements.txt index ecc55476..b32d9694 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,11 +47,11 @@ jsonschema==4.23.0 # via -r requirements.in jsonschema-specifications==2023.12.1 # via jsonschema -lxml==5.2.2 +lxml==5.3.0 # via # -r requirements.in # signxml -marshmallow==3.21.3 +marshmallow==3.22.0 # via -r requirements.in packaging==24.1 # via marshmallow diff --git a/src/cl_sii/__init__.py b/src/cl_sii/__init__.py index 0b50c67d..6436eb09 100644 --- a/src/cl_sii/__init__.py +++ b/src/cl_sii/__init__.py @@ -4,4 +4,4 @@ """ -__version__ = '0.33.0' +__version__ = '0.34.0' diff --git a/src/cl_sii/dte/data_models.py b/src/cl_sii/dte/data_models.py index c3a32a4e..73f1b9a4 100644 --- a/src/cl_sii/dte/data_models.py +++ b/src/cl_sii/dte/data_models.py @@ -19,6 +19,7 @@ from __future__ import annotations import dataclasses +import logging from datetime import date, datetime from typing import Mapping, Optional, Sequence @@ -33,6 +34,9 @@ from .constants import CodigoReferencia, TipoDte +logger = logging.getLogger(__name__) + + def validate_dte_folio(value: int) -> None: """ Validate value for DTE field ``folio``. @@ -99,6 +103,39 @@ def validate_non_empty_bytes(value: bytes) -> None: raise ValueError("Bytes value length is 0.") +VALIDATION_CONTEXT_TRUST_INPUT: str = 'trust_input' +""" +Key for the validation context to indicate that the input data is trusted. +""" + + +def is_input_trusted_according_to_validation_context( + validation_context: Optional[Mapping[str, object]] +) -> bool: + """ + Return whether the input data is trusted according to the validation context. + + :param validation_context: + The validation context of a Pydantic model. + Get it from ``pydantic.ValidationInfo.context``. + + Example for data classes: + + >>> dte_xml_data_instance_kwargs: Mapping[str, object] = dict( + ... emisor_rut=Rut('60910000-1'), # ... + ... ) + >>> dte_xml_data_adapter = pydantic.TypeAdapter(DteXmlData) + >>> dte_xml_data_instance: DteXmlData = dte_xml_data_adapter.validate_python( + ... dte_xml_data_instance_kwargs, + ... context={VALIDATION_CONTEXT_TRUST_INPUT: True} + ... ) + """ + if validation_context is None: + return False + else: + return validation_context.get(VALIDATION_CONTEXT_TRUST_INPUT) is True + + @pydantic.dataclasses.dataclass( frozen=True, config=pydantic.ConfigDict( @@ -815,7 +852,9 @@ def validate_referencias_numero_linea_ref_order(cls, v: object) -> object: return v @pydantic.model_validator(mode='after') - def validate_referencias_rut_otro_is_consistent_with_tipo_dte(self) -> DteXmlData: + def validate_referencias_rut_otro_is_consistent_with_tipo_dte( + self, info: pydantic.ValidationInfo + ) -> DteXmlData: referencias = self.referencias tipo_dte = self.tipo_dte @@ -826,27 +865,37 @@ def validate_referencias_rut_otro_is_consistent_with_tipo_dte(self) -> DteXmlDat ): for referencia in referencias: if referencia.rut_otro: - raise ValueError( + message: str = ( f"Setting a 'rut_otro' is not a valid option for this 'tipo_dte':" f" 'tipo_dte' == {tipo_dte!r}," - f" 'Referencia' number {referencia.numero_linea_ref}.", + f" 'Referencia' number {referencia.numero_linea_ref}." ) + if is_input_trusted_according_to_validation_context(info.context): + logger.warning('Validation failed but input is trusted: %s', message) + else: + raise ValueError(message) return self @pydantic.model_validator(mode='after') - def validate_referencias_rut_otro_is_consistent_with_emisor_rut(self) -> DteXmlData: + def validate_referencias_rut_otro_is_consistent_with_emisor_rut( + self, info: pydantic.ValidationInfo + ) -> DteXmlData: referencias = self.referencias emisor_rut = self.emisor_rut if isinstance(referencias, Sequence) and isinstance(emisor_rut, Rut): for referencia in referencias: if referencia.rut_otro and referencia.rut_otro == emisor_rut: - raise ValueError( + message: str = ( f"'rut_otro' must be different from 'emisor_rut':" f" {referencia.rut_otro!r} == {emisor_rut!r}," - f" 'Referencia' number {referencia.numero_linea_ref}.", + f" 'Referencia' number {referencia.numero_linea_ref}." ) + if is_input_trusted_according_to_validation_context(info.context): + logger.warning('Validation failed but input is trusted: %s', message) + else: + raise ValueError(message) return self diff --git a/src/cl_sii/dte/parse.py b/src/cl_sii/dte/parse.py index 2639ffa7..0b5ddf80 100644 --- a/src/cl_sii/dte/parse.py +++ b/src/cl_sii/dte/parse.py @@ -163,6 +163,7 @@ def parse_dte_xml(xml_doc: XmlElement) -> data_models.DteXmlData: 'ds:Signature', # "Firma Digital sobre Documento" namespaces=xml_utils.XML_DSIG_NS_MAP, ) + assert signature_em is not None if liquidacion_em is not None or exportaciones_em is not None: raise NotImplementedError("XML element 'Documento' is the only one supported.") @@ -191,6 +192,7 @@ def parse_dte_xml(xml_doc: XmlElement) -> data_models.DteXmlData: 'sii-dte:Encabezado', # "Identificacion y Totales del Documento" namespaces=DTE_XMLNS_MAP, ) + assert encabezado_em is not None # note: excluded because currently it is not useful. # ted_em = documento_em.find( # 'sii-dte:TED', # "Timbre Electronico de DTE" @@ -215,18 +217,22 @@ def parse_dte_xml(xml_doc: XmlElement) -> data_models.DteXmlData: 'sii-dte:IdDoc', # "Identificacion del DTE" namespaces=DTE_XMLNS_MAP, ) + assert id_doc_em is not None emisor_em = encabezado_em.find( 'sii-dte:Emisor', # "Datos del Emisor" namespaces=DTE_XMLNS_MAP, ) + assert emisor_em is not None receptor_em = encabezado_em.find( 'sii-dte:Receptor', # "Datos del Receptor" namespaces=DTE_XMLNS_MAP, ) + assert receptor_em is not None totales_em = encabezado_em.find( 'sii-dte:Totales', # "Montos Totales del DTE" namespaces=DTE_XMLNS_MAP, ) + assert totales_em is not None # 'Documento.Encabezado.IdDoc' # Excluded elements (optional according to the XML schema but the SII may require some of these @@ -453,6 +459,7 @@ def parse_dte_xml(xml_doc: XmlElement) -> data_models.DteXmlData: 'ds:KeyInfo', # "Informacion de Claves Publicas y Certificado" namespaces=xml_utils.XML_DSIG_NS_MAP, ) + assert signature_key_info_em is not None # signature_key_info_key_value_em = signature_key_info_em.find( # 'ds:KeyValue', # namespaces=xml_utils.XML_DSIG_NS_MAP) @@ -460,6 +467,7 @@ def parse_dte_xml(xml_doc: XmlElement) -> data_models.DteXmlData: 'ds:X509Data', # "Informacion del Certificado Publico" namespaces=xml_utils.XML_DSIG_NS_MAP, ) + assert signature_key_info_x509_data_em is not None signature_key_info_x509_cert_em = signature_key_info_x509_data_em.find( 'ds:X509Certificate', # "Certificado Publico" namespaces=xml_utils.XML_DSIG_NS_MAP, @@ -523,7 +531,7 @@ def parse_dte_xml(xml_doc: XmlElement) -> data_models.DteXmlData: ) -def _text_strip_or_none(xml_em: XmlElement) -> Optional[str]: +def _text_strip_or_none(xml_em: Optional[XmlElement]) -> Optional[str]: # note: we need the pair of functions '_text_strip_or_none' and '_text_strip_or_raise' # because, under certain circumstances, an XML tag: # - with no content -> `xml_em.text` is None instead of '' @@ -539,7 +547,7 @@ def _text_strip_or_none(xml_em: XmlElement) -> Optional[str]: return stripped_text -def _text_strip_or_raise(xml_em: XmlElement) -> str: +def _text_strip_or_raise(xml_em: Optional[XmlElement]) -> str: # note: we need the pair of functions '_text_strip_or_none' and '_text_strip_or_raise' # because, under certain circumstances, an XML tag: # - with no content -> `xml_em.text` is None instead of '' diff --git a/src/cl_sii/libs/xml_utils.py b/src/cl_sii/libs/xml_utils.py index c7b01354..da4be0fd 100644 --- a/src/cl_sii/libs/xml_utils.py +++ b/src/cl_sii/libs/xml_utils.py @@ -361,7 +361,11 @@ def verify_xml_signature( trusted_x509_cert: Optional[Union[crypto_utils.X509Cert, crypto_utils._X509CertOpenSsl]] = None, xml_verifier: Optional[signxml.verifier.XMLVerifier] = None, xml_verifier_supports_multiple_signatures: bool = False, -) -> Tuple[bytes, XmlElementTree, XmlElementTree]: +) -> Tuple[ + bytes, + Optional[Union[XmlElementTree, lxml.etree._Element]], + Union[XmlElementTree, lxml.etree._Element], +]: """ Verify the XML signature in ``xml_doc``. diff --git a/src/cl_sii/rtc/parse_aec.py b/src/cl_sii/rtc/parse_aec.py index 6591b10a..54ee4f96 100644 --- a/src/cl_sii/rtc/parse_aec.py +++ b/src/cl_sii/rtc/parse_aec.py @@ -69,14 +69,25 @@ def validate_aec_xml(xml_doc: XmlElement) -> None: xml_utils.validate_xml_doc(AEC_XML_SCHEMA_OBJ, xml_doc) -def parse_aec_xml(xml_doc: XmlElement) -> data_models_aec.AecXml: +def parse_aec_xml(xml_doc: XmlElement, trust_input: bool = False) -> data_models_aec.AecXml: """ Parse data from a "cesión"'s AEC XML doc. .. warning:: It is assumed that ``xml_doc`` is an ``{http://www.sii.cl/SiiDte}/AEC`` XML element. + + :param xml_doc: + AEC XML document. + :param trust_input: + If ``True``, the input data is trusted to be valid and + some validation errors are replaced by warnings. + + .. warning:: + Use this option *only* if you obtained the AEC XML document + from the SII *and* you need to work around some validation errors + that the SII should have caught, but let through. """ - aec_struct = _Aec.parse_xml(xml_doc) + aec_struct = _Aec.parse_xml(xml_doc, trust_input=trust_input) return aec_struct.as_aec_xml() @@ -134,11 +145,13 @@ def parse_xml_to_dict(xml_em: XmlElement) -> Mapping[str, object]: """ # XPath: //Signature/KeyInfo key_info_em = xml_em.find('ds:KeyInfo', namespaces=xml_utils.XML_DSIG_NS_MAP) + assert key_info_em is not None # XPath: //Signature/KeyInfo/X509Data key_info_x509_data_em = key_info_em.find( 'ds:X509Data', namespaces=xml_utils.XML_DSIG_NS_MAP ) + assert key_info_x509_data_em is not None # XPath: //Signature return dict( @@ -474,14 +487,17 @@ def parse_xml_to_dict(xml_em: XmlElement) -> Mapping[str, object]: """ # XPath: /AEC/DocumentoAEC/Cesiones/Cesion/DocumentoCesion/IdDTE id_dte_em = xml_em.find('sii-dte:IdDTE', namespaces=DTE_XMLNS_MAP) + assert id_dte_em is not None id_dte_dict = _IdDte.parse_xml_to_dict(id_dte_em) # XPath: /AEC/DocumentoAEC/Cesiones/Cesion/DocumentoCesion/Cedente cedente_em = xml_em.find('sii-dte:Cedente', namespaces=DTE_XMLNS_MAP) + assert cedente_em is not None cedente_dict = _Cedente.parse_xml_to_dict(cedente_em) # XPath: /AEC/DocumentoAEC/Cesiones/Cesion/DocumentoCesion/Cesionario cesionario_em = xml_em.find('sii-dte:Cesionario', namespaces=DTE_XMLNS_MAP) + assert cesionario_em is not None cesionario_dict = _Cesionario.parse_xml_to_dict(cesionario_em) # XPath: /AEC/DocumentoAEC/Cesiones/Cesion/DocumentoCesion @@ -543,6 +559,7 @@ def parse_xml_to_dict(xml_em: XmlElement) -> Mapping[str, object]: """ # XPath: /AEC/DocumentoAEC/Cesiones/Cesion/DocumentoCesion doc_cesion_em = xml_em.find('sii-dte:DocumentoCesion', namespaces=DTE_XMLNS_MAP) + assert doc_cesion_em is not None doc_cesion_dict = _DocumentoCesion.parse_xml_to_dict(doc_cesion_em) # Signature over 'DocumentoCesion' @@ -689,6 +706,7 @@ def parse_xml_to_dict(xml_em: XmlElement) -> Mapping[str, object]: 'sii-dte:DocumentoDTECedido', namespaces=DTE_XMLNS_MAP, ) + assert doc_dte_cedido_em is not None # Signature over 'DocumentoDTECedido' # XPath: /AEC/DocumentoAEC/Cesiones/DTECedido/Signature @@ -819,13 +837,16 @@ def parse_xml_to_dict(xml_em: XmlElement) -> Mapping[str, object]: """ # XPath: /AEC/DocumentoAEC/Caratula caratula_em = xml_em.find('sii-dte:Caratula', namespaces=DTE_XMLNS_MAP) + assert caratula_em is not None caratula_dict = _Caratula.parse_xml_to_dict(caratula_em) # XPath: /AEC/DocumentoAEC/Cesiones cesiones_em = xml_em.find('sii-dte:Cesiones', namespaces=DTE_XMLNS_MAP) + assert cesiones_em is not None # XPath: /AEC/DocumentoAEC/Cesiones/DTECedido dte_cedido_em = cesiones_em.find('sii-dte:DTECedido', namespaces=DTE_XMLNS_MAP) + assert dte_cedido_em is not None dte_cedido_dict = _DteCedido.parse_xml_to_dict(dte_cedido_em) # XPath: /AEC/DocumentoAEC/Cesiones/Cesion @@ -879,9 +900,14 @@ class _Aec(pydantic.BaseModel): ########################################################################### @classmethod - def parse_xml(cls, xml_doc: XmlElement) -> _Aec: + def parse_xml(cls, xml_doc: XmlElement, trust_input: bool = False) -> _Aec: aec_dict = cls.parse_xml_to_dict(xml_doc) - return cls.model_validate(aec_dict) + return cls.model_validate( + aec_dict, + context={ + cl_sii.dte.data_models.VALIDATION_CONTEXT_TRUST_INPUT: trust_input, + }, + ) def as_aec_xml(self) -> data_models_aec.AecXml: doc_aec_struct = self.documento_aec @@ -919,6 +945,7 @@ def parse_xml_to_dict(xml_doc: XmlElement) -> Mapping[str, object]: # XPath: /AEC/DocumentoAEC doc_aec_em = aec_em.find('sii-dte:DocumentoAEC', namespaces=DTE_XMLNS_MAP) + assert doc_aec_em is not None doc_aec_dict = _DocumentoAec.parse_xml_to_dict(doc_aec_em) # Signature over 'DocumentoAEC' @@ -927,6 +954,7 @@ def parse_xml_to_dict(xml_doc: XmlElement) -> Mapping[str, object]: 'ds:Signature', namespaces=xml_utils.XML_DSIG_NS_MAP, ) + assert signature_over_doc_aec_em is not None signature_over_doc_aec_dict = _XmlSignature.parse_xml_to_dict(signature_over_doc_aec_em) # XPath: /AEC diff --git a/src/tests/test_dte_data_models.py b/src/tests/test_dte_data_models.py index b37b1091..94f39fd2 100644 --- a/src/tests/test_dte_data_models.py +++ b/src/tests/test_dte_data_models.py @@ -2,6 +2,7 @@ import dataclasses import unittest from datetime import date, datetime +from typing import Mapping import pydantic @@ -13,6 +14,7 @@ TipoDte, ) from cl_sii.dte.data_models import ( # noqa: F401 + VALIDATION_CONTEXT_TRUST_INPUT, DteDataL0, DteDataL1, DteDataL2, @@ -1060,6 +1062,8 @@ def setUpClass(cls) -> None: 'test_data/sii-crypto/DTE--96670340-7--61--110616-cert.der' ) + cls.dte_xml_data_pydantic_type_adapter = pydantic.TypeAdapter(DteXmlData) + def setUp(self) -> None: super().setUp() @@ -1761,6 +1765,46 @@ def test_validate_referencias_rut_otro_is_consistent_with_tipo_dte(self) -> None self.assertEqual(len(validation_errors), len(expected_validation_errors)) self.assertEqual(validation_errors, expected_validation_errors) + def test_validate_referencias_rut_otro_is_consistent_with_tipo_dte_for_trusted_input( + self, + ) -> None: + obj = self.dte_xml_data_2 + obj_referencia = DteXmlReferencia( + numero_linea_ref=1, + tipo_documento_ref="801", + folio_ref="1", + fecha_ref=date(2019, 3, 28), + ind_global=None, + rut_otro=Rut('76354771-K'), + codigo_ref=None, + razon_ref=None, + ) + + expected_log_msg = ( + "Validation failed but input is trusted: " + "Setting a 'rut_otro' is not a valid option for this 'tipo_dte':" + " 'tipo_dte' == ," + " 'Referencia' number 1." + ) + + invalid_but_trusted_obj: Mapping[str, object] = { + **self.dte_xml_data_pydantic_type_adapter.dump_python(obj), + **dict( + referencias=[obj_referencia], + ), + } + validation_context = {VALIDATION_CONTEXT_TRUST_INPUT: True} + + try: + with self.assertLogs('cl_sii.dte.data_models', level='WARNING') as assert_logs_cm: + self.dte_xml_data_pydantic_type_adapter.validate_python( + invalid_but_trusted_obj, context=validation_context + ) + except pydantic.ValidationError as exc: + self.fail(f'{exc.__class__.__name__} raised') + + self.assertEqual(assert_logs_cm.records[0].getMessage(), expected_log_msg) + def test_validate_referencias_rut_otro_is_consistent_with_emisor_rut(self) -> None: obj = self.dte_xml_data_2 obj = dataclasses.replace( @@ -1805,6 +1849,50 @@ def test_validate_referencias_rut_otro_is_consistent_with_emisor_rut(self) -> No self.assertEqual(len(validation_errors), len(expected_validation_errors)) self.assertEqual(validation_errors, expected_validation_errors) + def test_validate_referencias_rut_otro_is_consistent_with_emisor_rut_for_trusted_input( + self, + ) -> None: + obj = self.dte_xml_data_2 + obj = dataclasses.replace( + obj, + tipo_dte=TipoDte.FACTURA_COMPRA_ELECTRONICA, + ) + obj_referencia = DteXmlReferencia( + numero_linea_ref=1, + tipo_documento_ref="801", + folio_ref="1", + fecha_ref=date(2019, 3, 28), + ind_global=None, + rut_otro=Rut('60910000-1'), + codigo_ref=None, + razon_ref=None, + ) + + expected_log_msg = ( + "Validation failed but input is trusted: " + "'rut_otro' must be different from 'emisor_rut':" + " Rut('60910000-1') == Rut('60910000-1')," + " 'Referencia' number 1." + ) + + invalid_but_trusted_obj: Mapping[str, object] = { + **self.dte_xml_data_pydantic_type_adapter.dump_python(obj), + **dict( + referencias=[obj_referencia], + ), + } + validation_context = {VALIDATION_CONTEXT_TRUST_INPUT: True} + + try: + with self.assertLogs('cl_sii.dte.data_models', level='WARNING') as assert_logs_cm: + self.dte_xml_data_pydantic_type_adapter.validate_python( + invalid_but_trusted_obj, context=validation_context + ) + except pydantic.ValidationError as exc: + self.fail(f'{exc.__class__.__name__} raised') + + self.assertEqual(assert_logs_cm.records[0].getMessage(), expected_log_msg) + def test_validate_referencias_codigo_ref_is_consistent_with_tipo_dte(self) -> None: obj = self.dte_xml_data_3 obj_referencia = dataclasses.replace(