From 6fd4f68de363dbf09f21815e1dd683dc96b7b2a4 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Aug 2025 22:54:55 +0300 Subject: [PATCH 1/6] fix: improve errors on backend errors --- src/datasets.jl | 1 + src/projects.jl | 1 + src/utils.jl | 25 +++++++++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/src/datasets.jl b/src/datasets.jl index 9b98dc9d1..cfd21ec84 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -695,6 +695,7 @@ function upload_dataset end # Any other 404 or other non-200 response indicates a backend failure _throw_invalidresponse(r) end + _check_internal_error(r; var="POST /user/datasets/{name}/versions") upload_config = _check_dataset_upload_config(r, dtype; newly_created_dataset) # Upload the actual data try diff --git a/src/projects.jl b/src/projects.jl index 903d06165..2069f8a99 100644 --- a/src/projects.jl +++ b/src/projects.jl @@ -241,6 +241,7 @@ function upload_project_dataset( # Other response codes indicate a backend failure _throw_invalidresponse(r) end + _check_internal_error(r; var="POST /user/datasets/{name}/versions") # ... upload_config = _check_dataset_upload_config(r, dtype; newly_created_dataset=false) # Upload the actual data diff --git a/src/utils.jl b/src/utils.jl index a03630234..905e2860b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -361,6 +361,31 @@ function _json_check_success(json::Dict; var::AbstractString) return nothing end +# Check that the API response is not a legacy 200 internal error, where +# we return +# +# {"success": false, "interal_error": true, "message": "..."} +# +# on internal errors. If it detects that this is an internal error, it throws +# a JuliaHubError. Returns `nothing` otherwise. +function _check_internal_error(r::_RESTResponse; var::AbstractString) + if !(r.status == 200) + return nothing + end + success = _get_json_or(r.json, "success", Any, nothing) + internal_error = _get_json_or(r.json, "internal_error", Any, nothing) + if (success === false) && (internal_error === true) + e = JuliaHubError( + """ + Internal Server Error 200 response from JuliaHub ($var): + JSON: $(sprint(show, MIME("text/plain"), r.json)) + """, + ) + throw(e) + end + return nothing +end + # Performs the print of f(::IO), but prepends `indent` spaces in front of # each line, to make it indented. function _print_indented(io::IO, f; indent::Integer) From 5aa79453d96998b64170bb93dd13b3de8d8063cf Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 26 Aug 2025 17:43:35 +0300 Subject: [PATCH 2/6] move, for ordering reasons --- src/restapi.jl | 25 +++++++++++++++++++++++++ src/utils.jl | 25 ------------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/restapi.jl b/src/restapi.jl index c44660ffd..5ebdb854c 100644 --- a/src/restapi.jl +++ b/src/restapi.jl @@ -79,6 +79,31 @@ function _parse_response_json(r::_RESTResponse, ::Type{T})::Tuple{T, String} whe return _parse_response_json(r.body, T) end +# Check that the API response is not a legacy 200 internal error, where +# we return +# +# {"success": false, "interal_error": true, "message": "..."} +# +# on internal errors. If it detects that this is an internal error, it throws +# a JuliaHubError. Returns `nothing` otherwise. +function _check_internal_error(r::_RESTResponse; var::AbstractString) + if !(r.status == 200) + return nothing + end + success = _get_json_or(r.json, "success", Any, nothing) + internal_error = _get_json_or(r.json, "internal_error", Any, nothing) + if (success === false) && (internal_error === true) + e = JuliaHubError( + """ + Internal Server Error 200 response from JuliaHub ($var): + JSON: $(sprint(show, MIME("text/plain"), r.json)) + """, + ) + throw(e) + end + return nothing +end + # _restcall calls _rest_request_mockable which calls _rest_request_http. The reason for this # indirection is that the signature of _rest_request_mockable is extremely simple and therefore # each to hook into with Mockable. diff --git a/src/utils.jl b/src/utils.jl index 905e2860b..a03630234 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -361,31 +361,6 @@ function _json_check_success(json::Dict; var::AbstractString) return nothing end -# Check that the API response is not a legacy 200 internal error, where -# we return -# -# {"success": false, "interal_error": true, "message": "..."} -# -# on internal errors. If it detects that this is an internal error, it throws -# a JuliaHubError. Returns `nothing` otherwise. -function _check_internal_error(r::_RESTResponse; var::AbstractString) - if !(r.status == 200) - return nothing - end - success = _get_json_or(r.json, "success", Any, nothing) - internal_error = _get_json_or(r.json, "internal_error", Any, nothing) - if (success === false) && (internal_error === true) - e = JuliaHubError( - """ - Internal Server Error 200 response from JuliaHub ($var): - JSON: $(sprint(show, MIME("text/plain"), r.json)) - """, - ) - throw(e) - end - return nothing -end - # Performs the print of f(::IO), but prepends `indent` spaces in front of # each line, to make it indented. function _print_indented(io::IO, f; indent::Integer) From fb7772c45f37f9ffb876e8935f679fc8e624ac44 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 28 Aug 2025 00:01:30 +0300 Subject: [PATCH 3/6] clean up, tests, version --- CHANGELOG.md | 6 ++++++ Project.toml | 2 +- src/datasets.jl | 5 +++-- src/projects.jl | 5 +++-- test/datasets.jl | 7 +++++++ test/mocking.jl | 11 +++++++++++ test/projects.jl | 4 ++++ 7 files changed, 35 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f76922d..62538f315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Version [v0.1.15] - 2025-08-28 + +### Fixed + +* The `JuliaHub.upload_dataset` now correctly throws a `JuliaHubError` on certain backend errors. ([#103]) + ## Version [v0.1.14] - 2025-06-11 ### Added diff --git a/Project.toml b/Project.toml index 49cad18bc..926b3a71c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "JuliaHub" uuid = "bc7fa6ce-b75e-4d60-89ad-56c957190b6e" authors = ["JuliaHub Inc."] -version = "0.1.14" +version = "0.1.15" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" diff --git a/src/datasets.jl b/src/datasets.jl index cfd21ec84..08be8b274 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -695,7 +695,6 @@ function upload_dataset end # Any other 404 or other non-200 response indicates a backend failure _throw_invalidresponse(r) end - _check_internal_error(r; var="POST /user/datasets/{name}/versions") upload_config = _check_dataset_upload_config(r, dtype; newly_created_dataset) # Upload the actual data try @@ -794,7 +793,9 @@ function _new_dataset( end function _open_dataset_version(auth::Authentication, name::AbstractString)::_RESTResponse - _restcall(auth, :POST, "user", "datasets", name, "versions") + r = _restcall(auth, :POST, "user", "datasets", name, "versions") + _check_internal_error(r; var="POST /user/datasets/{name}/versions") + return r end function _upload_dataset(upload_config, local_path; progress::Bool) diff --git a/src/projects.jl b/src/projects.jl index 2069f8a99..08ffa355f 100644 --- a/src/projects.jl +++ b/src/projects.jl @@ -241,7 +241,6 @@ function upload_project_dataset( # Other response codes indicate a backend failure _throw_invalidresponse(r) end - _check_internal_error(r; var="POST /user/datasets/{name}/versions") # ... upload_config = _check_dataset_upload_config(r, dtype; newly_created_dataset=false) # Upload the actual data @@ -283,12 +282,14 @@ function _open_dataset_version( auth::Authentication, dataset_uuid::UUID, project_uuid::UUID )::_RESTResponse body = Dict("project" => string(project_uuid)) - return JuliaHub._restcall( + r = JuliaHub._restcall( auth, :POST, ("datasets", string(dataset_uuid), "versions"), JSON.json(body), ) + _check_internal_error(r; var="POST /user/datasets/{name}/versions") + return r end function _close_dataset_version( diff --git a/test/datasets.jl b/test/datasets.jl index 4e3ff821e..0c9ed15b2 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -435,6 +435,13 @@ end @test_throws TypeError JuliaHub.upload_dataset( "example-dataset-license", @__FILE__; replace=true, license=(:fulltext, 1234) ) + + # Make sure we throw a JuliaHubError when we encounter an internal backend error + # that gets reported over a 200. + @test JuliaHub.upload_dataset("example-dataset-200-error-1", @__FILE__) isa JuliaHub.Dataset + MOCK_JULIAHUB_STATE[:internal_error_200] = true + @test_throws JuliaHub.JuliaHubError JuliaHub.upload_dataset("example-dataset-200-error", @__FILE__) + MOCK_JULIAHUB_STATE[:internal_error_200] = false end empty!(MOCK_JULIAHUB_STATE) end diff --git a/test/mocking.jl b/test/mocking.jl index 6a7a54ebf..794a33cb0 100644 --- a/test/mocking.jl +++ b/test/mocking.jl @@ -46,6 +46,14 @@ JuliaHub.__AUTH__[] = DEFAULT_GLOBAL_MOCK_AUTH Mocking.activate() const MOCK_JULIAHUB_STATE = Dict{Symbol, Any}() jsonresponse(status) = d -> JuliaHub._RESTResponse(status, JSON.json(d)) +function internal_error_200_response() + d = Dict( + "success" => false, + "internal_error" => true, + "message" => "Internal Server Error", + ) + return JuliaHub._RESTResponse(200, JSON.json(d)) +end mocking_patch = [ Mocking.@patch( JuliaHub._rest_request_mockable(args...; kwargs...) = _restcall_mocked(args...; kwargs...) @@ -412,6 +420,9 @@ function _restcall_mocked(method, url, headers, payload; query) Dict("repo_id" => string(UUIDs.uuid4())) |> jsonresponse(200) end elseif (method == :POST) && endswith(url, DATASET_VERSIONS_REGEX) + if get(MOCK_JULIAHUB_STATE, :internal_error_200, false) + return internal_error_200_response() + end dataset, is_user = let m = match(DATASET_VERSIONS_REGEX, url) URIs.unescapeuri(m[2]), m[1] == "user/" end diff --git a/test/projects.jl b/test/projects.jl index 4c3985f3a..306b1c1b6 100644 --- a/test/projects.jl +++ b/test/projects.jl @@ -243,6 +243,10 @@ end @test dataset.project.uuid === project_auth_2.project_id @test dataset.project.is_writable === false @test JuliaHub.upload_project_dataset(dataset_noproject, @__FILE__) isa JuliaHub.Dataset + + MOCK_JULIAHUB_STATE[:internal_error_200] = true + @test_throws JuliaHub.JuliaHubError JuliaHub.upload_project_dataset(dataset_noproject, @__FILE__) + MOCK_JULIAHUB_STATE[:internal_error_200] = false end end From 3d051b3c48951923b24fa4d35b71856550ef04df Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Thu, 28 Aug 2025 00:03:27 +0300 Subject: [PATCH 4/6] format --- test/datasets.jl | 4 +++- test/projects.jl | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/datasets.jl b/test/datasets.jl index 0c9ed15b2..f382ef23f 100644 --- a/test/datasets.jl +++ b/test/datasets.jl @@ -440,7 +440,9 @@ end # that gets reported over a 200. @test JuliaHub.upload_dataset("example-dataset-200-error-1", @__FILE__) isa JuliaHub.Dataset MOCK_JULIAHUB_STATE[:internal_error_200] = true - @test_throws JuliaHub.JuliaHubError JuliaHub.upload_dataset("example-dataset-200-error", @__FILE__) + @test_throws JuliaHub.JuliaHubError JuliaHub.upload_dataset( + "example-dataset-200-error", @__FILE__ + ) MOCK_JULIAHUB_STATE[:internal_error_200] = false end empty!(MOCK_JULIAHUB_STATE) diff --git a/test/projects.jl b/test/projects.jl index 306b1c1b6..346b076ac 100644 --- a/test/projects.jl +++ b/test/projects.jl @@ -245,7 +245,9 @@ end @test JuliaHub.upload_project_dataset(dataset_noproject, @__FILE__) isa JuliaHub.Dataset MOCK_JULIAHUB_STATE[:internal_error_200] = true - @test_throws JuliaHub.JuliaHubError JuliaHub.upload_project_dataset(dataset_noproject, @__FILE__) + @test_throws JuliaHub.JuliaHubError JuliaHub.upload_project_dataset( + dataset_noproject, @__FILE__ + ) MOCK_JULIAHUB_STATE[:internal_error_200] = false end end From 53dea8b3bd83ac0cdc8bc21aac3547b2cf5c5a7f Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Fri, 29 Aug 2025 20:24:55 +1200 Subject: [PATCH 5/6] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a87dff8bd..b5cafb221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Version [v0.1.15] - 2025-08-28 +## Version [v0.1.15] - 2025-08-29 ### Fixed From 89fea83bbdafcd46248ccc7cc832d914e36fa6f8 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Fri, 29 Aug 2025 11:26:14 +0300 Subject: [PATCH 6/6] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5cafb221..f03fb239f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -160,6 +160,7 @@ Initial package release. [v0.1.12]: https://github.com/JuliaComputing/JuliaHub.jl/releases/tag/v0.1.12 [v0.1.13]: https://github.com/JuliaComputing/JuliaHub.jl/releases/tag/v0.1.13 [v0.1.14]: https://github.com/JuliaComputing/JuliaHub.jl/releases/tag/v0.1.14 +[v0.1.15]: https://github.com/JuliaComputing/JuliaHub.jl/releases/tag/v0.1.15 [#1]: https://github.com/JuliaComputing/JuliaHub.jl/issues/1 [#2]: https://github.com/JuliaComputing/JuliaHub.jl/issues/2 [#3]: https://github.com/JuliaComputing/JuliaHub.jl/issues/3 @@ -195,3 +196,4 @@ Initial package release. [#96]: https://github.com/JuliaComputing/JuliaHub.jl/issues/96 [#99]: https://github.com/JuliaComputing/JuliaHub.jl/issues/99 [#100]: https://github.com/JuliaComputing/JuliaHub.jl/issues/100 +[#103]: https://github.com/JuliaComputing/JuliaHub.jl/issues/103