diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75b9ef04..0ec7f8e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,7 @@ jobs: - name: Run tests run: | # TODO: better way to disable all cloud backend tests? - pytest pins -m 'not fs_rsc and not fs_s3 and not fs_gcs and not skip_on_github' + pytest pins -m 'not fs_rsc and not fs_s3 and not fs_gcs and not fs_abfs and not skip_on_github' check-cross-compatibility: diff --git a/pins/drivers.py b/pins/drivers.py index e27a3865..06e631f1 100644 --- a/pins/drivers.py +++ b/pins/drivers.py @@ -105,6 +105,11 @@ def load_data( # TODO: update to handle multiple files return [str(Path(fs.open(path_to_file).name).absolute())] + elif meta.type == "json": + import json + + return json.load(fs.open(path_to_file)) + raise NotImplementedError(f"No driver for type {meta.type}") @@ -151,6 +156,12 @@ def save_data( import joblib joblib.dump(obj, final_name) + + elif type == "json": + import json + + json.dump(obj, open(final_name, "w")) + else: raise NotImplementedError(f"Cannot save type: {type}") diff --git a/pins/tests/test_boards.py b/pins/tests/test_boards.py index 5488f704..3f1c9aa2 100644 --- a/pins/tests/test_boards.py +++ b/pins/tests/test_boards.py @@ -128,7 +128,13 @@ def test_board_pin_write_rsc_index_html(board, tmp_dir2, snapshot): @parametrize( - "obj, type_", [(df, "csv"), (df, "joblib"), ({"a": 1, "b": [2, 3]}, "joblib")] + "obj, type_", + [ + (df, "csv"), + (df, "joblib"), + ({"a": 1, "b": [2, 3]}, "joblib"), + ({"a": 1, "b": [2, 3]}, "json"), + ], ) def test_board_pin_write_type(board, obj, type_, request): with rm_env(PINS_ENV_INSECURE_READ): diff --git a/pins/tests/test_drivers.py b/pins/tests/test_drivers.py index 0ec0bbe7..d382a182 100644 --- a/pins/tests/test_drivers.py +++ b/pins/tests/test_drivers.py @@ -77,6 +77,30 @@ def test_driver_roundtrip(tmp_dir2, type_): assert df.equals(obj) +@pytest.mark.parametrize( + "type_", + [ + "json", + ], +) +def test_driver_roundtrip_json(tmp_dir2, type_): + + df = {"x": [1, 2, 3]} + + fname = "some_df" + full_file = f"{fname}.{type_}" + + p_obj = tmp_dir2 / fname + res_fname = save_data(df, p_obj, type_) + + assert Path(res_fname).name == full_file + + meta = MetaRaw(full_file, type_, "my_pin") + obj = load_data(meta, fsspec.filesystem("file"), tmp_dir2, allow_pickle_read=True) + + assert df == obj + + def test_driver_feather_write_error(tmp_dir2): import pandas as pd