Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: improve new minio implementation #546

Merged
merged 3 commits into from
Mar 3, 2025
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
fix: address feedback
  • Loading branch information
joseph-sentry committed Feb 28, 2025
commit 21612749d083511b4b38a7371aa6df26fce6f4c2
10 changes: 2 additions & 8 deletions shared/storage/__init__.py
Original file line number Diff line number Diff line change
@@ -10,17 +10,11 @@ def get_appropriate_storage_service(
repoid: int | None = None,
force_minio=False,
) -> BaseStorageService:
return get_minio_storage_service(repoid)


def get_minio_storage_service(
repo_id: int | None,
) -> MinioStorageService:
minio_config = get_config("services", "minio", default={})
if repo_id:
if repoid:
new_minio_mode = cast(
Literal["read", "write"] | None,
NEW_MINIO.check_value(repo_id, default=None), # type: ignore
NEW_MINIO.check_value(repoid, default=None), # type: ignore
)
return MinioStorageService(
minio_config,
8 changes: 4 additions & 4 deletions shared/storage/minio.py
Original file line number Diff line number Diff line change
@@ -133,6 +133,9 @@ def get_cached_minio_client(
)


zstd_default = zstd_decoded_by_default()


# Service class for interfacing with codecov's underlying storage layer, minio
class MinioStorageService(BaseStorageService, PresignedURLService):
def __init__(
@@ -143,17 +146,13 @@ def __init__(
self.minio_config = minio_config
self.new_read = new_mode == "read" or new_mode == "write"
self.new_write = new_mode == "write"
self.zstd_default = zstd_decoded_by_default()

log.debug("Connecting to minio with config %s", self.minio_config)

self.minio_client = get_cached_minio_client(**self.minio_config)

log.debug("Done setting up minio client")

def client(self):
return self.minio_client if self.minio_client else None

# writes the initial storage bucket to storage via minio.
def create_root_storage(self, bucket_name="archive", region="us-east-1"):
read_only_policy = {
@@ -239,6 +238,7 @@ def read_file(
bucket_name,
path,
file_obj,
zstd_default,
)
else:
return old_minio_read(
20 changes: 7 additions & 13 deletions tests/unit/storage/test_init.py
Original file line number Diff line number Diff line change
@@ -21,6 +21,8 @@ def test_get_appropriate_storage_service_minio(self, mock_configuration):
res = get_appropriate_storage_service()
assert isinstance(res, MinioStorageService)
assert res.minio_config == minio_config
assert not res.new_read
assert not res.new_write

def test_get_appropriate_storage_service_read_new_minio(
self, mock_configuration, mocker
@@ -30,12 +32,8 @@ def test_get_appropriate_storage_service_read_new_minio(
"minio": minio_config,
}
mocker.patch(
"shared.storage.READ_NEW_MINIO.check_value",
return_value=True,
)
mocker.patch(
"shared.storage.WRITE_NEW_MINIO.check_value",
return_value=False,
"shared.storage.NEW_MINIO.check_value",
return_value="read",
)
res = get_appropriate_storage_service(repoid=123)
assert isinstance(res, MinioStorageService)
@@ -51,15 +49,11 @@ def test_get_appropriate_storage_service_write_new_minio(
"minio": minio_config,
}
mocker.patch(
"shared.storage.WRITE_NEW_MINIO.check_value",
return_value=True,
)
mocker.patch(
"shared.storage.READ_NEW_MINIO.check_value",
return_value=False,
"shared.storage.NEW_MINIO.check_value",
return_value="write",
)
res = get_appropriate_storage_service(repoid=123)
assert isinstance(res, MinioStorageService)
assert res.minio_config == minio_config
assert res.new_read
assert res.new_write
assert not res.new_read
3 changes: 1 addition & 2 deletions tests/unit/storage/test_new_minio.py
Original file line number Diff line number Diff line change
@@ -39,8 +39,7 @@ def make_storage() -> MinioStorageService:
"iam_auth": False,
"iam_endpoint": None,
},
new_read=True,
new_write=True,
new_mode="write",
)


Loading
Oops, something went wrong.