diff --git a/lua/copy_build.lua b/lua/copy_build.lua index 70af142..43825f1 100644 --- a/lua/copy_build.lua +++ b/lua/copy_build.lua @@ -65,29 +65,40 @@ else return end -local total_number_of_objects = 0 +local objects = {} for object in res.body:gmatch("([^\r\n]+)[\r\n]+") do - total_number_of_objects = total_number_of_objects + 1 + table.insert(objects, object) end + +local total_number_of_objects = #objects +local batch_size = 16 local current_object = 0 -for object in res.body:gmatch("([^\r\n]+)[\r\n]+") do - local object_url, object_res - current_object = current_object + 1 - ngx.say("[" .. current_object .. "/" .. total_number_of_objects .. "] Copying " .. object .. " ... ") - ngx.flush(true) +for batch_start = 1, total_number_of_objects, batch_size do + local batch_end = math.min(batch_start + batch_size - 1, total_number_of_objects) + local urls = {} + for i = batch_start, batch_end do + table.insert(urls, { + "/force_real_request/copy/" .. build_src .. "/" .. build_tgt .. "/" .. objects[i], + { method = ngx.HTTP_PUT, body = '' } + }) + end - object_url = "/force_real_request/copy/" .. build_src .. "/" .. build_tgt .. "/" .. object - object_res = ngx.location.capture(object_url, { method = ngx.HTTP_PUT, body = '' }) + local results = { ngx.location.capture_multi(urls) } - if object_res.status == 200 then - ngx.say('DONE') - ngx.flush(true) - else - ngx.say('FAILED') - ngx.flush(true) - return + for i, object_res in ipairs(results) do + current_object = current_object + 1 + local object = objects[batch_start + i - 1] + ngx.say("[" .. current_object .. "/" .. total_number_of_objects .. "] " .. object .. " ... ") + if object_res.status == 200 then + ngx.say('DONE') + else + ngx.say('FAILED') + ngx.flush(true) + return + end end + ngx.flush(true) end ngx.say("BUILD COPIED") diff --git a/tests/end2end/test_copy.py b/tests/end2end/test_copy.py index 0284780..03149bb 100644 --- a/tests/end2end/test_copy.py +++ b/tests/end2end/test_copy.py @@ -2,7 +2,7 @@ import pytest -from constants import STAGING_BUILD +from constants import STAGING_BUILD, PROMOTED_BUILD COPY_BUILD = f'copy_of_{STAGING_BUILD}' @@ -71,6 +71,34 @@ def test_copy_fails_when_target_already_exists( assert lines[-1] == b'FAILED' +def test_copy_promotes_staging_to_promoted_bucket( + session, artifacts_url, upload_file, finish_build +): + """Copy from a staging build to a promoted build (cross-bucket promotion). + + Uses 19 objects to exercise the multi-batch loop (batch_size=16), and + embeds the index in each object's content to catch cross-object regressions. + """ + n = 19 + for i in range(n): + upload_file(STAGING_BUILD, f'obj-{i}', f'content-{i}'.encode()) + finish_build(STAGING_BUILD) + + resp = session.get(f'{artifacts_url}/copy/{STAGING_BUILD}/{PROMOTED_BUILD}/') + assert resp.status_code == 200 + assert resp.content.splitlines()[-1] == b'BUILD COPIED' + + # Verify every promoted object has the correct content. + for i in range(n): + dl = session.get(f'{artifacts_url}/download/{PROMOTED_BUILD}/obj-{i}') + assert dl.status_code == 200, f'obj-{i} not found in promoted build' + assert dl.content == f'content-{i}'.encode(), f'obj-{i} has unexpected content' + + # Verify promotion metadata files are present. + assert session.get(f'{artifacts_url}/download/{PROMOTED_BUILD}/.final_status').status_code == 200 + assert session.get(f'{artifacts_url}/download/{PROMOTED_BUILD}/.original_build').status_code == 200 + + def test_copy_behind_ingress(session, artifacts_url, upload_file, finish_build): """Copy works correctly when a Script-Name ingress header is present.""" upload_file(STAGING_BUILD, '.final_status', b'SUCCESSFUL',)