Skip to content

Commit

Permalink
Add test for parse_json_file and change typing to os.PathLike (#30183)
Browse files Browse the repository at this point in the history
* Add test for parse_json_file

* Change Path to PathLike

* Fix `Import block is un-sorted or un-formatted`

* revert parse_json_file

* Fix ruff format

* Add parse_json_file test
  • Loading branch information
xu-song authored and Ita Zaporozhets committed May 14, 2024
1 parent 9685033 commit d292063
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 3 deletions.
9 changes: 7 additions & 2 deletions src/transformers/hf_argparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import dataclasses
import json
import os
import sys
import types
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError
Expand Down Expand Up @@ -376,7 +377,9 @@ def parse_dict(self, args: Dict[str, Any], allow_extra_keys: bool = False) -> Tu
raise ValueError(f"Some keys are not used by the HfArgumentParser: {sorted(unused_keys)}")
return tuple(outputs)

def parse_json_file(self, json_file: Union[str, Path], allow_extra_keys: bool = False) -> Tuple[DataClass, ...]:
def parse_json_file(
self, json_file: Union[str, os.PathLike], allow_extra_keys: bool = False
) -> Tuple[DataClass, ...]:
"""
Alternative helper method that does not use `argparse` at all, instead loading a json file and populating the
dataclass types.
Expand All @@ -398,7 +401,9 @@ def parse_json_file(self, json_file: Union[str, Path], allow_extra_keys: bool =
outputs = self.parse_dict(data, allow_extra_keys=allow_extra_keys)
return tuple(outputs)

def parse_yaml_file(self, yaml_file: Union[str, Path], allow_extra_keys: bool = False) -> Tuple[DataClass, ...]:
def parse_yaml_file(
self, yaml_file: Union[str, os.PathLike], allow_extra_keys: bool = False
) -> Tuple[DataClass, ...]:
"""
Alternative helper method that does not use `argparse` at all, instead loading a yaml file and populating the
dataclass types.
Expand Down
2 changes: 1 addition & 1 deletion tests/utils/test_hf_argparser.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ def test_parse_json(self):
os.mkdir(temp_local_path)
with open(temp_local_path + ".json", "w+") as f:
json.dump(args_dict_for_json, f)
parsed_args = parser.parse_yaml_file(Path(temp_local_path + ".json"))[0]
parsed_args = parser.parse_json_file(Path(temp_local_path + ".json"))[0]

args = BasicExample(**args_dict_for_json)
self.assertEqual(parsed_args, args)
Expand Down

0 comments on commit d292063

Please sign in to comment.