NotRequired keys of TypedDict are incorrectly inferred as required #570
Description
🐛 Bug report
See the documentation for TypedDict for context.
Note that by default, a TypedDict requires all keys to be present, i.e., it is equivalent to being created with total=True as a metaclass argument.
One may specify individual keys as not required with the NotRequired annotation.
This annotation is not respected by jsonargparse.
Namely, this check in jsonargparse._typehints does not consider the possibility of per-field Required/NotRequired annotations.
To reproduce
from dataclasses import dataclass, field
from jsonargparse import ActionConfigFile, ArgumentParser, lazy_instance
import tempfile
from typing import TypedDict, NotRequired
import yaml
class TestDict(TypedDict):
a: int
b: NotRequired[int]
@dataclass
class TestClass:
test: TestDict = field(default_factory=lambda: TestDict(a=0))
parser = ArgumentParser(exit_on_error=False)
parser.add_argument(
"-c",
"--config",
action=ActionConfigFile,
help="Path to a configuration file in json or yaml format.")
parser.add_class_arguments(
TestClass,
"test",
fail_untyped=False,
instantiate=True,
sub_configs=True,
default=lazy_instance(
TestClass,
),
)
config = yaml.safe_dump({
"test": {
"test": {
"a": 1
}
}
})
with tempfile.NamedTemporaryFile("w", suffix=".yaml") as f:
f.write(config)
f.flush()
cfg = parser.parse_args(["--config", f"{f.name}"])Expected behavior
The script should exit without error after successfully parsing the arguments.
Actual behavior
Traceback
Traceback (most recent call last):
File ".../lib/python3.11/site-packages/jsonargparse/_typehints.py", line 581, in _check_type
raise ex
File ".../lib/python3.11/site-packages/jsonargparse/_typehints.py", line 566, in _check_type
val = adapt_typehints(val, self._typehint, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_typehints.py", line 913, in adapt_typehints
raise_unexpected_value(f"Missing required keys: {missing_keys}", val)
File ".../lib/python3.11/site-packages/jsonargparse/_typehints.py", line 702, in raise_unexpected_value
raise ValueError(message) from exception
ValueError: Missing required keys: {'b'}. Got value: {'a': 1}
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 605, in parse_string
cfg = self._load_config_parser_mode(cfg_str, cfg_path, ext_vars, previous_config.get())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 654, in _load_config_parser_mode
return self._apply_actions(cfg_dict, prev_cfg=prev_cfg)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 1314, in _apply_actions
value = self._check_value_key(action, value, action_dest, prev_cfg)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 1367, in _check_value_key
value = action._check_type_(value, cfg=cfg) # type: ignore[attr-defined]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_common.py", line 268, in _check_type_
return self._check_type(value, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_typehints.py", line 591, in _check_type
raise TypeError(f'Parser key "{self.dest}"{elem}:\n{error}') from ex
TypeError: Parser key "test.test":
Missing required keys: {'b'}. Got value: {'a': 1}
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 264, in parse_known_args
namespace, args = self._parse_known_args(args, namespace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/argparse.py", line 2114, in _parse_known_args
start_index = consume_optional(start_index)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/argparse.py", line 2054, in consume_optional
take_action(action, args, option_string)
File ".../lib/python3.11/argparse.py", line 1978, in take_action
action(self, namespace, argument_values, option_string)
File ".../lib/python3.11/site-packages/jsonargparse/_actions.py", line 170, in __call__
self.apply_config(parser, cfg, self.dest, values)
File ".../lib/python3.11/site-packages/jsonargparse/_actions.py", line 205, in apply_config
cfg_file = parser.parse_path(value, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 562, in parse_path
parsed_cfg = self.parse_string(
^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_deprecated.py", line 123, in patched_parse
cfg = parse_method(*args, _skip_check=_skip_check, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 621, in parse_string
self.error(str(ex), ex)
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 1010, in error
raise argument_error(message) from ex
argparse.ArgumentError: Parser key "test.test":
Missing required keys: {'b'}. Got value: {'a': 1}
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
...
File "...", line 140, in <module>
cfg = parser.parse_args(["--config", f"{f.name}"])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_deprecated.py", line 123, in patched_parse
cfg = parse_method(*args, _skip_check=_skip_check, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 396, in parse_args
cfg, unk = self.parse_known_args(args=args, namespace=cfg)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 266, in parse_known_args
self.error(str(ex), ex)
File ".../lib/python3.11/site-packages/jsonargparse/_core.py", line 1010, in error
raise argument_error(message) from ex
argparse.ArgumentError: Parser key "test.test":
Missing required keys: {'b'}. Got value: {'a': 1}Environment
- jsonargparse version: 4.32.0
- Python version: 3.11.6
- How jsonargparse was installed:
pip - OS: Ubuntu 20.04