From fa07ecadb5492b7f232e9b0e29a585b942fb7755 Mon Sep 17 00:00:00 2001 From: Hongyang Peng Date: Wed, 15 May 2024 10:17:15 +0800 Subject: [PATCH] docs:update docs for new env of s3 (#362) --- README.md | 4 ++-- docs/configuration/s3.md | 26 ++++++++++++++++++-------- megfile/s3_path.py | 3 ++- tests/test_s3.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f658e410..0f04f86d 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,10 @@ Using `s3` as an example, the following describes the configuration methods. For You can use environments and configuration file for configuration, and priority is that environment variables take precedence over configuration file. ### Use environments -You can use environments to setup authentication credentials for your s3 account: +You can use environments to setup authentication credentials for your `s3` account: - `AWS_ACCESS_KEY_ID`: access key - `AWS_SECRET_ACCESS_KEY`: secret key -- `OSS_ENDPOINT`: endpoint url of s3 +- `OSS_ENDPOINT` / `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL`: endpoint url of s3 - `AWS_S3_ADDRESSING_STYLE`: addressing style ### Use command diff --git a/docs/configuration/s3.md b/docs/configuration/s3.md index 33852d34..c0a61532 100644 --- a/docs/configuration/s3.md +++ b/docs/configuration/s3.md @@ -5,7 +5,7 @@ You can use environments and configuration file for configuration, and priority You can use environments to setup authentication credentials for your s3 account: - `AWS_ACCESS_KEY_ID`: access key - `AWS_SECRET_ACCESS_KEY`: secret key -- `OSS_ENDPOINT`: endpoint url of s3 +- `OSS_ENDPOINT` / `AWS_ENDPOINT_URL_S3` / `AWS_ENDPOINT_URL`: endpoint url of s3 - `AWS_S3_ADDRESSING_STYLE`: addressing style ### Use command @@ -13,10 +13,10 @@ You can update config file with `megfile` command easyly: [megfile config s3 [OPTIONS] AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY](https://megvii-research.github.io/megfile/cli.html#megfile-config-s3) ```bash -$ megfile config s3 accesskey secretkey +$ megfile config s3 access_key secret_key # for aliyun -$ megfile config s3 accesskey secretkey \ +$ megfile config s3 access_key secret_key \ --addressing-style virtual \ --endpoint-url http://oss-cn-hangzhou.aliyuncs.com \ ``` @@ -24,14 +24,24 @@ $ megfile config s3 accesskey secretkey \ You can get the configuration from `~/.aws/credentials`, like: ```ini [default] -aws_secret_access_key = accesskey -aws_access_key_id = secretkey +aws_secret_access_key = access_key +aws_access_key_id = secret_key s3 = addressing_style = virtual endpoint_url = http://oss-cn-hangzhou.aliyuncs.com ``` +Megfile also support a custom global endpoint url, if you not configure a endpoint url for S3 in the file. Example: + +```ini +[default] +aws_secret_access_key = access_key +aws_access_key_id = secret_key + +endpoint_url = http://oss-cn-hangzhou.aliyuncs.com +``` + ### Config for different s3 server or authentications You can operate s3 files with different endpoint urls, access keys and secret keys. For example, you have two s3 server with different endpoint_url, access_key and secret key. With configuration, you can use path with profile name like `s3+profile_name://bucket/key` to operate different s3 server: @@ -51,13 +61,13 @@ You need use `PROFILE_NAME__` prefix, like: #### Using command: ```bash -megfile config s3 AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY --profile-name profile1 +megfile config s3 access_key secret_key --profile-name profile1 ``` Then the config file's content will be: ```ini [profile1] -aws_secret_access_key = accesskey -aws_access_key_id = secretkey +aws_access_key_id = access_key +aws_secret_access_key = secret_key ``` diff --git a/megfile/s3_path.py b/megfile/s3_path.py index efe33f60..303baee3 100644 --- a/megfile/s3_path.py +++ b/megfile/s3_path.py @@ -141,7 +141,8 @@ def get_endpoint_url(profile_name: Optional[str] = None) -> str: if profile_name: environ_keys = (f'{profile_name}__OSS_ENDPOINT'.upper(),) else: - environ_keys = ('OSS_ENDPOINT', 'AWS_ENDPOINT_URL') + environ_keys = ( + 'OSS_ENDPOINT', 'AWS_ENDPOINT_URL_S3', 'AWS_ENDPOINT_URL') for environ_key in environ_keys: environ_endpoint_url = os.environ.get(environ_key) if environ_endpoint_url: diff --git a/tests/test_s3.py b/tests/test_s3.py index 60966f23..05f9ac5d 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -256,17 +256,52 @@ def test_get_endpoint_url(mocker): def test_get_endpoint_url_from_env(mocker): mocker.patch('megfile.s3_path.get_scoped_config', return_value={}) mocker.patch.dict(os.environ, {'OSS_ENDPOINT': 'oss-endpoint'}) + assert s3.get_endpoint_url() == 'oss-endpoint' + + +def test_get_endpoint_url_from_env2(mocker): + mocker.patch('megfile.s3_path.get_scoped_config', return_value={}) + mocker.patch.dict(os.environ, {'AWS_ENDPOINT_URL': 'oss-endpoint2'}) + assert s3.get_endpoint_url() == 'oss-endpoint2' + +def test_get_endpoint_url_from_env3(mocker): + mocker.patch('megfile.s3_path.get_scoped_config', return_value={}) + mocker.patch.dict( + os.environ, { + 'OSS_ENDPOINT': 'oss-endpoint', + 'AWS_ENDPOINT_URL': 'oss-endpoint2' + }) assert s3.get_endpoint_url() == 'oss-endpoint' +def test_get_endpoint_url_from_env4(mocker): + mocker.patch('megfile.s3_path.get_scoped_config', return_value={}) + mocker.patch.dict(os.environ, {'AWS_ENDPOINT_URL_S3': 'oss-endpoint3'}) + assert s3.get_endpoint_url() == 'oss-endpoint3' + + def test_get_endpoint_url_from_scoped_config(mocker): mocker.patch( 'megfile.s3_path.get_scoped_config', return_value={'s3': { 'endpoint_url': 'test_endpoint_url' }}) + assert s3.get_endpoint_url() == 'test_endpoint_url' + + mocker.patch( + 'megfile.s3_path.get_scoped_config', + return_value={'endpoint_url': 'test_endpoint_url2'}) + assert s3.get_endpoint_url() == 'test_endpoint_url2' + mocker.patch( + 'megfile.s3_path.get_scoped_config', + return_value={ + 's3': { + 'endpoint_url': 'test_endpoint_url' + }, + 'endpoint_url': 'test_endpoint_url2' + }) assert s3.get_endpoint_url() == 'test_endpoint_url'