file_not_in_storage error while do-upload call from github actions to self hosted setup #35
Description
Sharing the logs for the details of error from the worker when codecov cli makes call from the github actions for do-upload
{"message": "Calling update to Commit.Report", "asctime": "2024-03-15 07:25:35,038", "name": "services.report", "levelname": "INFO", "lineno": 939, "pathname": "/worker/services/report/__init__.py", "funcName": "save_report", "threadName": "MainThread", "size": 232, "ownerid": 12, "repoid": 475, "commitid": "b464eacd141042cdc167f03bde1de47c5ee5d665", "task_name": "app.tasks.upload.UploadProcessor", "task_id": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "logger.name": "services.report", "logger.thread_name": "MainThread", "level": "INFO"}
{"message": "Calling update to reports_reportdetails.files_array", "asctime": "2024-03-15 07:25:35,041", "name": "services.report", "levelname": "INFO", "lineno": 960, "pathname": "/worker/services/report/__init__.py", "funcName": "save_report", "threadName": "MainThread", "size": 56, "ownerid": 12, "repoid": 475, "commitid": "b464eacd141042cdc167f03bde1de47c5ee5d665", "task_name": "app.tasks.upload.UploadProcessor", "task_id": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "logger.name": "services.report", "logger.thread_name": "MainThread", "level": "INFO"}
{"message": "Archived report", "asctime": "2024-03-15 07:25:35,053", "name": "services.report", "levelname": "INFO", "lineno": 976, "pathname": "/worker/services/report/__init__.py", "funcName": "save_report", "threadName": "MainThread", "repoid": 475, "commit": "b464eacd141042cdc167f03bde1de47c5ee5d665", "url": "v4/repos/000D745F22D21AAC36C6473D22191256/commits/b464eacd141042cdc167f03bde1de47c5ee5d665/chunks.txt", "number_sessions": 0, "new_report_sessions": {}, "task_name": "app.tasks.upload.UploadProcessor", "task_id": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "logger.name": "services.report", "logger.thread_name": "MainThread", "level": "INFO"}
{"message": "Processed 0 reports", "asctime": "2024-03-15 07:25:35,057", "name": "tasks.upload_processor", "levelname": "INFO", "lineno": 260, "pathname": "/worker/tasks/upload_processor.py", "funcName": "process_async_within_lock", "threadName": "MainThread", "repoid": 475, "commit": "b464eacd141042cdc167f03bde1de47c5ee5d665", "commit_yaml": {"codecov": {"require_ci_to_pass": true}, "coverage": {"precision": 2, "round": "down", "range": [60.0, 80.0], "status": {"project": true, "patch": true, "changes": false, "default_rules": {"flag_coverage_not_uploaded_behavior": "include"}}}, "comment": {"layout": "reach,diff,flags,tree", "behavior": "default", "show_carryforward_flags": false}, "slack_app": true, "github_checks": {"annotations": true}}, "url": "v4/repos/000D745F22D21AAC36C6473D22191256/commits/b464eacd141042cdc167f03bde1de47c5ee5d665/chunks.txt", "parent_task": "fe9ba3fd-3407-4338-b255-6d5ec2a33b7a", "task_name": "app.tasks.upload.UploadProcessor", "task_id": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "logger.name": "tasks.upload_processor", "logger.thread_name": "MainThread", "level": "INFO"}
{"message": "Task app.tasks.upload.UploadProcessor[dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654] succeeded in 0.5057511939999131s: {'processings_so_far': [{'arguments': {...}, 'successful': False, 'error': {...}, 'report': None, 'should_retry': False}]}", "asctime": "2024-03-15 07:25:35,067", "name": "celery.app.trace", "levelname": "INFO", "lineno": 131, "pathname": "/usr/local/lib/python3.10/site-packages/celery/app/trace.py", "funcName": "info", "threadName": "MainThread", "data": {"id": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "name": "app.tasks.upload.UploadProcessor", "return_value": "{'processings_so_far': [{'arguments': {...}, 'successful': False, 'error': {...}, 'report': None, 'should_retry': False}]}", "runtime": 0.5057511939999131, "args": "[{}]", "kwargs": "{'repoid': 475, 'commitid': 'b464eacd141042cdc167f03bde1de47c5ee5d665', 'commit_yaml': {'codecov': {'require_ci_to_pass': True}, 'coverage': {'precision': 2, 'round': 'down', 'range': [60.0, 80.0], 'status': {'project': True, 'patch': True, 'changes': False, 'default_rules': {'flag_coverage_not_uploaded_behavior': 'include'}}}, 'comment': {'layout': 'reach,diff,flags,tree', 'behavior': 'default', 'show_carryforward_flags': False}, 'slack_app': True, 'github_checks': {'annotations': True}}, 'arguments_list': [{'commit': 'b464eacd141042cdc167f03bde1de47c5ee5d665', 'upload_id': 29, 'version': 'v4', 'report_code': None, 'reportid': '3b043166-f549-4689-be2e-280e2b517b9e', 'upload_pk': 29}], 'report_code': None}"}, "task_name": "app.tasks.upload.UploadProcessor", "task_id": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "logger.name": "celery.app.trace", "logger.thread_name": "MainThread", "level": "INFO"}
{"message": "Task app.tasks.upload.UploadFinisher[909cd653-98f1-42b3-a3f8-c40eb6e7fc1d] received", "asctime": "2024-03-15 07:25:35,068", "name": "celery.worker.strategy", "levelname": "INFO", "lineno": 161, "pathname": "/usr/local/lib/python3.10/site-packages/celery/worker/strategy.py", "funcName": "task_message_handler", "threadName": "MainThread", "data": {"id": "909cd653-98f1-42b3-a3f8-c40eb6e7fc1d", "name": "app.tasks.upload.UploadFinisher", "args": "[{'processings_so_far': [{'arguments': {'commit': 'b464eacd141042cdc167f03bde1de47c5ee5d665', 'upload_id': 29, 'version': 'v4', 'report_code': None, 'reportid': '3b043166-f549-4689-be2e-280e2b517b9e', 'upload_pk': 29}, 'successful': False, 'error': {'code': 'file_not_in_storage', 'params': {'location': 'v4/raw/2024-03-15/000D745F22D21AAC36C6473D22191256/b464eacd141042cdc167f03bde1de47c5ee5d665/3b043166-f549-4689-be2e-280e2b517b9e/f822ba3b-e3f1-4efb-9a8a-1901f9420902.txt'}}, 'report': None, 'should_retry': False}]}]", "kwargs": "{'repoid': 475, 'commitid': 'b464eacd141042cdc167f03bde1de47c5ee5d665', 'commit_yaml': {'codecov': {'require_ci_to_pass': True}, 'coverage': {'precision': 2, 'round': 'down', 'range': [60.0, 80.0], 'status': {'project': True, 'patch': True, 'changes': False, 'default_rules': {'flag_coverage_not_uploaded_behavior': 'include'}}}, 'comment': {'layout': 'reach,diff,flags,tree', 'behavior': 'default', 'show_carryforward_flags': False}, 'slack_app': True, 'github_checks': {'annotations': True}}, 'report_code': None, 'checkpoints_UploadFlow': {'UPLOAD_TASK_BEGIN': 1710487514288, 'PROCESSING_BEGIN': 1710487514307, 'INITIAL_PROCESSING_COMPLETE': 1710487514324}}"}, "task_name": "???", "task_id": "???", "logger.name": "celery.worker.strategy", "logger.thread_name": "MainThread", "level": "INFO"}
{"message": "Received upload_finisher task", "asctime": "2024-03-15 07:25:35,069", "name": "tasks.upload_finisher", "levelname": "INFO", "lineno": 64, "pathname": "/worker/tasks/upload_finisher.py", "funcName": "run_async", "threadName": "MainThread", "repoid": 475, "commit": "b464eacd141042cdc167f03bde1de47c5ee5d665", "processing_results": {"processings_so_far": [{"arguments": {"commit": "b464eacd141042cdc167f03bde1de47c5ee5d665", "upload_id": 29, "version": "v4", "report_code": null, "reportid": "3b043166-f549-4689-be2e-280e2b517b9e", "upload_pk": 29}, "successful": false, "error": {"code": "file_not_in_storage", "params": {"location": "v4/raw/2024-03-15/000D745F22D21AAC36C6473D22191256/b464eacd141042cdc167f03bde1de47c5ee5d665/3b043166-f549-4689-be2e-280e2b517b9e/f822ba3b-e3f1-4efb-9a8a-1901f9420902.txt"}}, "report": null, "should_retry": false}]}, "parent_task": "dce6fd0a-59a3-4bc6-8fb3-f42ba9f15654", "task_name": "app.tasks.upload.UploadFinisher", "task_id": "909cd653-98f1-42b3-a3f8-c40eb6e7fc1d", "logger.name": "tasks.upload_finisher", "logger.thread_name": "MainThread", "level": "INFO"}
The actions finally fails with the error AccessDeniedAccess Denied
There are other files present in S3 like commit/chunk.txt , comparision.json ruling out the problem of S3 connectivity .
Not sure if this is related to codecov/engineering-team#128 @scott-codecov @trent-codecov