Skip to content

Nightly spec-drift: strict contract tests failing (since 2026-09-01) #511

Description

@github-actions

Nightly strict contract tests failed

The scheduled Spec Drift Detection run found the SDK contract suite no longer matches upstream OpenAPI/AsyncAPI.

To resolve: locally run uv run python scripts/sync_spec.py + uv run python scripts/generate.py, reconcile models/maps, and open a PR with Closes #<this issue>. This tracker auto-closes on the next green scheduled run.

To silence while a reconcile is in flight, add the spec-drift-ack label — nightly runs then stop commenting until the failing set changes.

Failing test output (last 200 lines)

    
        schema = _resolve_request_body_schema(
            self.spec,
            entry.path_template,
            entry.http_method,
        )
        assert schema is not None, (
            f"Spec has no body schema for {entry.http_method} {entry.path_template}"
        )
    
        errors: list[str] = []
        _check_model_drift(
            model_cls=model_cls,
            model_fqn=model_fqn,
            spec_schema=schema,
            errors=errors,
        )
    
        # Issue #52: also drift-check nested BaseModel fields (one level deep).
        for field_name, nested_cls, nested_schema in _iter_nested_body_models(
            model_cls,
            schema,
            self.spec,
        ):
            nested_fqn = f"{nested_cls.__module__}.{nested_cls.__qualname__}"
            _check_model_drift(
                model_cls=nested_cls,
                model_fqn=nested_fqn,
                spec_schema=nested_schema,
                errors=errors,
                context=f"{model_fqn}.{field_name}",
            )
    
        if errors:
>           pytest.fail(f"{model_fqn} <-> {entry.request_body_schema}\n" + "\n".join(errors))
E           Failed: kalshi.models.api_keys.CreateApiKeyRequest <-> #/components/schemas/CreateApiKeyRequest
E           [ADD drift] spec has properties SDK model missing: ['fcm_subtrader_id']

tests/test_contracts.py:1718: Failed
________ TestRequestBodyDrift.test_body_properties_match_spec[generate] ________

self = <tests.test_contracts.TestRequestBodyDrift object at 0x7f452d242810>
entry = MethodEndpointEntry(sdk_method='kalshi.resources.api_keys.ApiKeysResource.generate', http_method='POST', path_template='/api_keys/generate', request_body_schema='#/components/schemas/GenerateApiKeyRequest')

    def test_body_properties_match_spec(
        self,
        entry: MethodEndpointEntry,
    ) -> None:
        assert entry.request_body_schema is not None
        model_fqn = BODY_MODEL_MAP.get(entry.request_body_schema)
        assert model_fqn is not None, (
            f"No request model registered in BODY_MODEL_MAP for "
            f"{entry.request_body_schema!r}. Add the mapping."
        )
        model_cls = _get_model_class_from_fqn(model_fqn)
    
        schema = _resolve_request_body_schema(
            self.spec,
            entry.path_template,
            entry.http_method,
        )
        assert schema is not None, (
            f"Spec has no body schema for {entry.http_method} {entry.path_template}"
        )
    
        errors: list[str] = []
        _check_model_drift(
            model_cls=model_cls,
            model_fqn=model_fqn,
            spec_schema=schema,
            errors=errors,
        )
    
        # Issue #52: also drift-check nested BaseModel fields (one level deep).
        for field_name, nested_cls, nested_schema in _iter_nested_body_models(
            model_cls,
            schema,
            self.spec,
        ):
            nested_fqn = f"{nested_cls.__module__}.{nested_cls.__qualname__}"
            _check_model_drift(
                model_cls=nested_cls,
                model_fqn=nested_fqn,
                spec_schema=nested_schema,
                errors=errors,
                context=f"{model_fqn}.{field_name}",
            )
    
        if errors:
>           pytest.fail(f"{model_fqn} <-> {entry.request_body_schema}\n" + "\n".join(errors))
E           Failed: kalshi.models.api_keys.GenerateApiKeyRequest <-> #/components/schemas/GenerateApiKeyRequest
E           [ADD drift] spec has properties SDK model missing: ['fcm_subtrader_id']

tests/test_contracts.py:1718: Failed
_ TestRequestBodyDrift.test_body_properties_match_spec[set_target_balance_allocation] _

self = <tests.test_contracts.TestRequestBodyDrift object at 0x7f452d243710>
entry = MethodEndpointEntry(sdk_method='kalshi.resources.portfolio.PortfolioResource.set_target_balance_allocation', http_meth...e='/portfolio/target_balance_allocation', request_body_schema='#/components/schemas/SetTargetBalanceAllocationRequest')

    def test_body_properties_match_spec(
        self,
        entry: MethodEndpointEntry,
    ) -> None:
        assert entry.request_body_schema is not None
        model_fqn = BODY_MODEL_MAP.get(entry.request_body_schema)
        assert model_fqn is not None, (
            f"No request model registered in BODY_MODEL_MAP for "
            f"{entry.request_body_schema!r}. Add the mapping."
        )
        model_cls = _get_model_class_from_fqn(model_fqn)
    
        schema = _resolve_request_body_schema(
            self.spec,
            entry.path_template,
            entry.http_method,
        )
        assert schema is not None, (
            f"Spec has no body schema for {entry.http_method} {entry.path_template}"
        )
    
        errors: list[str] = []
        _check_model_drift(
            model_cls=model_cls,
            model_fqn=model_fqn,
            spec_schema=schema,
            errors=errors,
        )
    
        # Issue #52: also drift-check nested BaseModel fields (one level deep).
        for field_name, nested_cls, nested_schema in _iter_nested_body_models(
            model_cls,
            schema,
            self.spec,
        ):
            nested_fqn = f"{nested_cls.__module__}.{nested_cls.__qualname__}"
            _check_model_drift(
                model_cls=nested_cls,
                model_fqn=nested_fqn,
                spec_schema=nested_schema,
                errors=errors,
                context=f"{model_fqn}.{field_name}",
            )
    
        if errors:
>           pytest.fail(f"{model_fqn} <-> {entry.request_body_schema}\n" + "\n".join(errors))
E           Failed: kalshi.models.portfolio.SetTargetBalanceAllocationRequest <-> #/components/schemas/SetTargetBalanceAllocationRequest
E           [ADD drift] spec has properties SDK model missing: ['resting_margin_reservation']

tests/test_contracts.py:1718: Failed
=============================== warnings summary ===============================
tests/test_contracts.py::TestSpecDrift::test_additive_drift[SetTargetBalanceAllocationRequest]
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:837: AdditiveOptionalDriftWarning: Additive-optional drift in kalshi.models.portfolio.SetTargetBalanceAllocationRequest: Spec field 'resting_margin_reservation' has no SDK mapping
    _warn_additive_optional(entry.sdk_model, additive_optional)

tests/test_contracts.py::TestSpecDrift::test_additive_drift[ApiKey]
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:837: AdditiveOptionalDriftWarning: Additive-optional drift in kalshi.models.api_keys.ApiKey: Spec field 'fcm_subtrader_id' has no SDK mapping
    _warn_additive_optional(entry.sdk_model, additive_optional)

tests/test_contracts.py::TestSpecDrift::test_additive_drift[CreateApiKeyRequest]
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:837: AdditiveOptionalDriftWarning: Additive-optional drift in kalshi.models.api_keys.CreateApiKeyRequest: Spec field 'fcm_subtrader_id' has no SDK mapping
    _warn_additive_optional(entry.sdk_model, additive_optional)

tests/test_contracts.py::TestSpecDrift::test_additive_drift[CreateApiKeyResponse]
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:837: AdditiveOptionalDriftWarning: Additive-optional drift in kalshi.models.api_keys.CreateApiKeyResponse: Spec field 'warning' has no SDK mapping
    _warn_additive_optional(entry.sdk_model, additive_optional)

tests/test_contracts.py::TestSpecDrift::test_additive_drift[GenerateApiKeyRequest]
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:837: AdditiveOptionalDriftWarning: Additive-optional drift in kalshi.models.api_keys.GenerateApiKeyRequest: Spec field 'fcm_subtrader_id' has no SDK mapping
    _warn_additive_optional(entry.sdk_model, additive_optional)

tests/test_contracts.py::TestSpecDrift::test_additive_drift[GenerateApiKeyResponse]
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:837: AdditiveOptionalDriftWarning: Additive-optional drift in kalshi.models.api_keys.GenerateApiKeyResponse: Spec field 'warning' has no SDK mapping
    _warn_additive_optional(entry.sdk_model, additive_optional)

tests/test_contracts.py::TestAdditiveDriftClassification::test_optional_drift_survives_the_nightly_strict_gate
  /home/runner/work/kalshi-python-sdk/kalshi-python-sdk/tests/test_contracts.py:988: AdditiveOptionalDriftWarning: Additive-optional drift in tests.synthetic.Fake: Spec field 'opt_new' has no SDK mapping
    _warn_additive_optional("tests.synthetic.Fake", msgs)

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED tests/test_contracts.py::test_every_spec_endpoint_is_mapped[core] - Failed: core: spec operations not in the endpoint map (add a MethodEndpointEntry, or record in _UNIMPLEMENTED_ENDPOINTS with a reason):
  - GET /live_data/weather/{city}/calibrations
FAILED tests/test_contracts.py::test_every_spec_endpoint_is_mapped[perps_scm] - Failed: perps_scm: spec operations not in the endpoint map (add a MethodEndpointEntry, or record in _UNIMPLEMENTED_ENDPOINTS with a reason):
  - DELETE /fcm/margin/api_keys/{api_key_id}
  - GET /fcm/margin/api_keys
  - POST /fcm/margin/api_keys
  - POST /fcm/margin/api_keys/generate
FAILED tests/test_contracts.py::TestRequestParamDrift::test_sync_params_match_spec[list1] - Failed: kalshi.resources.api_keys.ApiKeysResource.list <-> GET /api_keys
[ADD drift] spec has params SDK missing: ['fcm_subtrader_id']
(Allowlist via EXCLUSIONS[('kalshi.resources.api_keys.ApiKeysResource.list', '...')])
FAILED tests/test_contracts.py::TestRequestParamDrift::test_async_params_match_spec[list1] - Failed: kalshi.resources.api_keys.AsyncApiKeysResource.list <-> GET /api_keys
[ADD drift] spec has params SDK missing: ['fcm_subtrader_id']
(Allowlist via EXCLUSIONS[('kalshi.resources.api_keys.ApiKeysResource.list', '...')])
FAILED tests/test_contracts.py::TestRequestBodyDrift::test_body_properties_match_spec[create0] - Failed: kalshi.models.api_keys.CreateApiKeyRequest <-> #/components/schemas/CreateApiKeyRequest
[ADD drift] spec has properties SDK model missing: ['fcm_subtrader_id']
FAILED tests/test_contracts.py::TestRequestBodyDrift::test_body_properties_match_spec[generate] - Failed: kalshi.models.api_keys.GenerateApiKeyRequest <-> #/components/schemas/GenerateApiKeyRequest
[ADD drift] spec has properties SDK model missing: ['fcm_subtrader_id']
FAILED tests/test_contracts.py::TestRequestBodyDrift::test_body_properties_match_spec[set_target_balance_allocation] - Failed: kalshi.models.portfolio.SetTargetBalanceAllocationRequest <-> #/components/schemas/SetTargetBalanceAllocationRequest
[ADD drift] spec has properties SDK model missing: ['resting_margin_reservation']
============ 7 failed, 738 passed, 7 warnings in 282.20s (0:04:42) =============

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    spec-driftUpstream OpenAPI/AsyncAPI spec changed since last sync

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions