Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #51976 from trociny/wip-59065-quincy
quincy: rgw: fix multipart upload object leaks due to re-upload Reviewed-by: Casey Bodley <cbodley@redhat.com>
- Loading branch information
Showing
37 changed files
with
2,803 additions
and
2,095 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| tasks: | ||
| - workunit: | ||
| clients: | ||
| client.0: | ||
| - rgw/test_rgw_s3_mp_reupload.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| import boto3 | ||
| import botocore.exceptions | ||
| import sys | ||
| import os | ||
| import subprocess | ||
|
|
||
| #boto3.set_stream_logger(name='botocore') | ||
|
|
||
| # handles two optional system arguments: | ||
| # <bucket-name> : default is "bkt134" | ||
| # <0 or 1> : 0 -> upload aborted, 1 -> completed; default is completed | ||
|
|
||
| if len(sys.argv) >= 2: | ||
| bucket_name = sys.argv[1] | ||
| else: | ||
| bucket_name = "bkt314738362229" | ||
| print("bucket nams is %s" % bucket_name) | ||
|
|
||
| complete_mpu = True | ||
| if len(sys.argv) >= 3: | ||
| complete_mpu = int(sys.argv[2]) > 0 | ||
|
|
||
| versioned_bucket = False | ||
| if len(sys.argv) >= 4: | ||
| versioned_bucket = int(sys.argv[3]) > 0 | ||
|
|
||
| rgw_host = os.environ['RGW_HOST'] | ||
| access_key = os.environ['RGW_ACCESS_KEY'] | ||
| secret_key = os.environ['RGW_SECRET_KEY'] | ||
|
|
||
| try: | ||
| endpoint='http://%s:%d' % (rgw_host, 80) | ||
| client = boto3.client('s3', | ||
| endpoint_url=endpoint, | ||
| aws_access_key_id=access_key, | ||
| aws_secret_access_key=secret_key) | ||
| res = client.create_bucket(Bucket=bucket_name) | ||
| except botocore.exceptions.EndpointConnectionError: | ||
| try: | ||
| endpoint='https://%s:%d' % (rgw_host, 443) | ||
| client = boto3.client('s3', | ||
| endpoint_url=endpoint, | ||
| verify=False, | ||
| aws_access_key_id=access_key, | ||
| aws_secret_access_key=secret_key) | ||
| res = client.create_bucket(Bucket=bucket_name) | ||
| except botocore.exceptions.EndpointConnectionError: | ||
| endpoint='http://%s:%d' % (rgw_host, 8000) | ||
| client = boto3.client('s3', | ||
| endpoint_url=endpoint, | ||
| aws_access_key_id=access_key, | ||
| aws_secret_access_key=secret_key) | ||
| res = client.create_bucket(Bucket=bucket_name) | ||
|
|
||
| print("endpoint is %s" % endpoint) | ||
|
|
||
| if versioned_bucket: | ||
| res = client.put_bucket_versioning( | ||
| Bucket=bucket_name, | ||
| VersioningConfiguration={ | ||
| 'MFADelete': 'Disabled', | ||
| 'Status': 'Enabled'} | ||
| ) | ||
|
|
||
| key = "mpu_test4" | ||
| nparts = 2 | ||
| ndups = 11 | ||
| do_reupload = True | ||
|
|
||
| part_path = "/tmp/mp_part_5m" | ||
| subprocess.run(["dd", "if=/dev/urandom", "of=" + part_path, "bs=1M", "count=5"], check=True) | ||
|
|
||
| f = open(part_path, 'rb') | ||
|
|
||
| res = client.create_multipart_upload(Bucket=bucket_name, Key=key) | ||
| mpu_id = res["UploadId"] | ||
|
|
||
| print("start UploadId=%s" % (mpu_id)) | ||
|
|
||
| parts = [] | ||
| parts2 = [] | ||
|
|
||
| for ix in range(0,nparts): | ||
| part_num = ix + 1 | ||
| f.seek(0) | ||
| res = client.upload_part(Body=f, Bucket=bucket_name, Key=key, | ||
| UploadId=mpu_id, PartNumber=part_num) | ||
| # save | ||
| etag = res['ETag'] | ||
| part = {'ETag': etag, 'PartNumber': part_num} | ||
| print("phase 1 uploaded part %s" % part) | ||
| parts.append(part) | ||
|
|
||
| if do_reupload: | ||
| # just re-upload part 1 | ||
| part_num = 1 | ||
| for ix in range(0,ndups): | ||
| f.seek(0) | ||
| res = client.upload_part(Body=f, Bucket=bucket_name, Key=key, | ||
| UploadId=mpu_id, PartNumber=part_num) | ||
| etag = res['ETag'] | ||
| part = {'ETag': etag, 'PartNumber': part_num} | ||
| print ("phase 2 uploaded part %s" % part) | ||
|
|
||
| # save | ||
| etag = res['ETag'] | ||
| part = {'ETag': etag, 'PartNumber': part_num} | ||
| parts2.append(part) | ||
|
|
||
| if complete_mpu: | ||
| print("completing multipart upload, parts=%s" % parts) | ||
| res = client.complete_multipart_upload( | ||
| Bucket=bucket_name, Key=key, UploadId=mpu_id, | ||
| MultipartUpload={'Parts': parts}) | ||
| else: | ||
| print("aborting multipart upload, parts=%s" % parts) | ||
| res = client.abort_multipart_upload( | ||
| Bucket=bucket_name, Key=key, UploadId=mpu_id) | ||
|
|
||
| # clean up | ||
| subprocess.run(["rm", "-f", part_path], check=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # INITIALIZATION | ||
|
|
||
| mydir=$(dirname $0) | ||
| data_pool=default.rgw.buckets.data | ||
| orphan_list_out=/tmp/orphan_list.$$ | ||
| radoslist_out=/tmp/radoslist.$$ | ||
| rados_ls_out=/tmp/rados_ls.$$ | ||
| diff_out=/tmp/diff.$$ | ||
|
|
||
| rgw_host="$(hostname --fqdn)" | ||
| echo "INFO: fully qualified domain name: $rgw_host" | ||
|
|
||
| export RGW_ACCESS_KEY="0555b35654ad1656d804" | ||
| export RGW_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==" | ||
| export RGW_HOST="${RGW_HOST:-$rgw_host}" | ||
|
|
||
| # random argument determines if multipart is aborted or completed 50/50 | ||
| outcome=$((RANDOM % 2)) | ||
| if [ $outcome -eq 0 ] ;then | ||
| echo "== TESTING *ABORTING* MULTIPART UPLOAD WITH RE-UPLOADS ==" | ||
| else | ||
| echo "== TESTING *COMPLETING* MULTIPART UPLOAD WITH RE-UPLOADS ==" | ||
| fi | ||
|
|
||
| # random argument determines if multipart is aborted or completed 50/50 | ||
| versioning=$((RANDOM % 2)) | ||
| if [ $versioning -eq 0 ] ;then | ||
| echo "== TESTING NON-VERSIONED BUCKET ==" | ||
| else | ||
| echo "== TESTING VERSIONED BUCKET ==" | ||
| fi | ||
|
|
||
| # create a randomized bucket name | ||
| bucket="reupload-bkt-$((RANDOM % 899999 + 100000))" | ||
|
|
||
|
|
||
| # SET UP PYTHON VIRTUAL ENVIRONMENT | ||
|
|
||
| # install boto3 | ||
| python3 -m venv $mydir | ||
| source $mydir/bin/activate | ||
| pip install pip --upgrade | ||
| pip install boto3 | ||
|
|
||
|
|
||
| # CREATE RGW USER IF NECESSARY | ||
|
|
||
| if radosgw-admin user info --access-key $RGW_ACCESS_KEY 2>/dev/null ;then | ||
| echo INFO: user already exists | ||
| else | ||
| echo INFO: creating user | ||
| radosgw-admin user create --uid testid \ | ||
| --access-key $RGW_ACCESS_KEY \ | ||
| --secret $RGW_SECRET_KEY \ | ||
| --display-name 'M. Tester' \ | ||
| --email tester@ceph.com 2>/dev/null | ||
| fi | ||
|
|
||
|
|
||
| # RUN REUPLOAD TEST | ||
|
|
||
| $mydir/bin/python3 ${mydir}/test_rgw_s3_mp_reupload.py $bucket $outcome $versioning | ||
|
|
||
|
|
||
| # ANALYZE FOR ERRORS | ||
| # (NOTE: for now we're choosing not to use the rgw-orphan-list tool) | ||
|
|
||
| # force garbage collection to remove extra parts | ||
| radosgw-admin gc process --include-all 2>/dev/null | ||
|
|
||
| marker=$(radosgw-admin metadata get bucket:$bucket 2>/dev/null | grep bucket_id | sed 's/.*: "\(.*\)".*/\1/') | ||
|
|
||
| # determine expected rados objects | ||
| radosgw-admin bucket radoslist --bucket=$bucket 2>/dev/null | sort >$radoslist_out | ||
| echo "radosgw-admin bucket radoslist:" | ||
| cat $radoslist_out | ||
|
|
||
| # determine found rados objects | ||
| rados ls -p $data_pool 2>/dev/null | grep "^$marker" | sort >$rados_ls_out | ||
| echo "rados ls:" | ||
| cat $rados_ls_out | ||
|
|
||
| # compare expected and found | ||
| diff $radoslist_out $rados_ls_out >$diff_out | ||
| if [ $(cat $diff_out | wc -l) -ne 0 ] ;then | ||
| error=1 | ||
| echo "ERROR: Found differences between expected and actual rados objects for test bucket." | ||
| echo " note: indicators: '>' found but not expected; '<' expected but not found." | ||
| cat $diff_out | ||
| fi | ||
|
|
||
|
|
||
| # CLEAN UP | ||
|
|
||
| deactivate | ||
|
|
||
| rm -f $orphan_list_out $radoslist_out $rados_ls_out $diff_out | ||
|
|
||
|
|
||
| # PRODUCE FINAL RESULTS | ||
|
|
||
| if [ -n "$error" ] ;then | ||
| echo "== FAILED ==" | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "== PASSED ==" | ||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.