[azure-storage-blob] Blob uploads and downloads give authentication error when certain characters are in the blob path #11028

sebastiaanzaaijer · 2020-04-23T19:31:00Z

Package Name: azure-storage-blob
Package Version: 12.3.0
Operating System: Ubuntu Linux 16.04.6
Python Version: 3.8.2

Describe the bug
An authentication error "The MAC signature found in the HTTP request '' is not the same as any computed signature. Server used following string to sign: ''" occurs when the following characters are in the blob path name:

! $ & ' ( ) * , : ; = @

To Reproduce
Steps to reproduce the behavior:

Blob storage with or without data lake
Create an empty container (e.g. test)
Upload/create a blob to a path with special character (e.g. /test/test=/test.txt)
You will receive an authentication error

Expected behavior
Upload should work as expected. Creating the same folder structure in the portal or locally and then uploading using the azcopy tool works.

Additional context
Example code, to make it run you need the following:

empty container called test
pupulated container called azcopy with the following directory structure:

/test`/test.txt
/test^/test.txt
/test~/test.txt
/test</test.txt
/test=/test.txt
/test>/test.txt
/test|/test.txt
/test_/test.txt
/test-/test.txt
/test,/test.txt
/test;/test.txt
/test:/test.txt
/test!/test.txt
/test?/test.txt
/test./test.txt
/test'/test.txt
/test"/test.txt
/test(/test.txt
/test)/test.txt
/test[/test.txt
/test]/test.txt
/test{/test.txt
/test}/test.txt
/test@/test.txt
/test$/test.txt
/test*/test.txt
/test&/test.txt
/test#/test.txt
/test%/test.txt
/test+/test.txt
/test2=/test.txt

import os
import asyncio
from azure.core.exceptions import ClientAuthenticationError

TEST_FILE = "test.txt"
UPLOAD_CONTAINER_NAME = "test"
DOWNLOAD_CONTAINER_NAME = "azcopytest"

connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")


async def create_blob_in_folder(folder, debug_info=False):

    from azure.storage.blob.aio import BlobServiceClient

    blob_service_client = BlobServiceClient.from_connection_string(connection_string)

    try:
        async with blob_service_client:
            container_client = blob_service_client.get_container_client(
                UPLOAD_CONTAINER_NAME
            )

            with open(TEST_FILE, "rb") as data:
                blob_client = container_client.get_blob_client(
                    "{}/test.txt".format(folder)
                )
                await blob_client.upload_blob(data, blob_type="BlockBlob")
    except ClientAuthenticationError as e:
        print("uploading to folder: {}".format(folder), "Authentication error")
        if debug_info:
            print(e)
    else:
        print("uploading to folder: {}".format(folder), "Success")


async def download_blob_from_folder(folder, debug_info=False):

    from azure.storage.blob.aio import BlobServiceClient

    blob_service_client = BlobServiceClient.from_connection_string(connection_string)

    try:
        async with blob_service_client:
            container_client = blob_service_client.get_container_client(
                DOWNLOAD_CONTAINER_NAME
            )

            blob_client = container_client.get_blob_client("{}/test.txt".format(folder))
            stream = await blob_client.download_blob()
            data = await stream.readall()
    except ClientAuthenticationError as e:
        print("uploading to folder: {}".format(folder), "Authentication error")
        if debug_info:
            print(e)
    else:
        print("downloading from folder: {}".format(folder), "Success")


async def main():
    await create_blob_in_folder("test!")
    await create_blob_in_folder('test"')
    await create_blob_in_folder("test#")
    await create_blob_in_folder("test$")
    await create_blob_in_folder("test%")
    await create_blob_in_folder("test&")
    await create_blob_in_folder("test'")
    await create_blob_in_folder("test(")
    await create_blob_in_folder("test)")
    await create_blob_in_folder("test*")
    await create_blob_in_folder("test+")
    await create_blob_in_folder("test,")
    await create_blob_in_folder("test-")
    await create_blob_in_folder("test.")
    await create_blob_in_folder("test:")
    await create_blob_in_folder("test;")
    await create_blob_in_folder("test<")
    await create_blob_in_folder("test=")
    await create_blob_in_folder("test>")
    await create_blob_in_folder("test?")
    await create_blob_in_folder("test@")
    await create_blob_in_folder("test[")
    await create_blob_in_folder("test]")
    await create_blob_in_folder("test^")
    await create_blob_in_folder("test_")
    await create_blob_in_folder("test`")
    await create_blob_in_folder("test{")
    await create_blob_in_folder("test|")
    await create_blob_in_folder("test}")
    await create_blob_in_folder("test~")
    await create_blob_in_folder("test2=", True)

    await download_blob_from_folder("test!")
    await download_blob_from_folder('test"')
    await download_blob_from_folder("test#")
    await download_blob_from_folder("test$")
    await download_blob_from_folder("test%")
    await download_blob_from_folder("test&")
    await download_blob_from_folder("test'")
    await download_blob_from_folder("test(")
    await download_blob_from_folder("test)")
    await download_blob_from_folder("test*")
    await download_blob_from_folder("test+")
    await download_blob_from_folder("test,")
    await download_blob_from_folder("test-")
    await download_blob_from_folder("test.")
    await download_blob_from_folder("test:")
    await download_blob_from_folder("test;")
    await download_blob_from_folder("test<")
    await download_blob_from_folder("test=")
    await download_blob_from_folder("test>")
    await download_blob_from_folder("test?")
    await download_blob_from_folder("test@")
    await download_blob_from_folder("test[")
    await download_blob_from_folder("test]")
    await download_blob_from_folder("test^")
    await download_blob_from_folder("test_")
    await download_blob_from_folder("test`")
    await download_blob_from_folder("test{")
    await download_blob_from_folder("test|")
    await download_blob_from_folder("test}")
    await download_blob_from_folder("test~")
    await download_blob_from_folder("test2=", True)


if __name__ == "__main__":
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())

Output:

uploading to folder: test! Authentication error
uploading to folder: test" Success
uploading to folder: test# Success
uploading to folder: test$ Authentication error
uploading to folder: test% Success
uploading to folder: test& Authentication error
uploading to folder: test' Authentication error
uploading to folder: test( Authentication error
uploading to folder: test) Authentication error
uploading to folder: test* Authentication error
uploading to folder: test+ Success
uploading to folder: test, Authentication error
uploading to folder: test- Success
uploading to folder: test. Success
uploading to folder: test: Authentication error
uploading to folder: test; Authentication error
uploading to folder: test< Success
uploading to folder: test= Authentication error
uploading to folder: test> Success
uploading to folder: test? Success
uploading to folder: test@ Authentication error
uploading to folder: test[ Success
uploading to folder: test] Success
uploading to folder: test^ Success
uploading to folder: test_ Success
uploading to folder: test` Success
uploading to folder: test{ Success
uploading to folder: test| Success
uploading to folder: test} Success
uploading to folder: test~ Success
uploading to folder: test2= Authentication error
Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.
RequestId:<snip>
Time:<snip>
ErrorCode:AuthenticationFailed
Error:None
AuthenticationErrorDetail:The MAC signature found in the HTTP request '<snip>' is not the same as any computed signature. Server used following string to sign: '<snip>'.
downloading from folder: test! Authentication error
downloading from folder: test" Success
downloading from folder: test# Success
downloading from folder: test$ Authentication error
downloading from folder: test% Success
downloading from folder: test& Authentication error
downloading from folder: test' Authentication error
downloading from folder: test( Authentication error
downloading from folder: test) Authentication error
downloading from folder: test* Authentication error
downloading from folder: test+ Success
downloading from folder: test, Authentication error
downloading from folder: test- Success
downloading from folder: test. Success
downloading from folder: test: Authentication error
downloading from folder: test; Authentication error
downloading from folder: test< Success
downloading from folder: test= Authentication error
downloading from folder: test> Success
downloading from folder: test? Success
downloading from folder: test@ Authentication error
downloading from folder: test[ Success
downloading from folder: test] Success
downloading from folder: test^ Success
downloading from folder: test_ Success
downloading from folder: test` Success
downloading from folder: test{ Success
downloading from folder: test| Success
downloading from folder: test} Success
downloading from folder: test~ Success
downloading from folder: test2= Authentication error
Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.
RequestId:<snip>
Time:<snip>
ErrorCode:AuthenticationFailed
Error:None
AuthenticationErrorDetail:The MAC signature found in the HTTP request '<snip>' is not the same as any computed signature. Server used following string to sign: '<snip>'.

The text was updated successfully, but these errors were encountered:

ghost · 2020-04-23T20:18:01Z

Thanks for the feedback! We are routing this to the appropriate team for follow-up. cc @xgithubtriage.

amishra-dev · 2020-04-24T04:59:38Z

@sebastiaanzaaijer you should consider url encoding the blob names, that will get you out of this error the fastest. Let me know how it goes.
I do agree that the error is not ideal, we will look at improving it.

sebastiaanzaaijer · 2020-04-24T05:37:41Z

@amishra-dev thank you very much for prompt resposnse and the suggestion. Url encoding the path does work, however then the blob gets stored in it's url encoded path rather than the intended one (see below). I'm trying to create a file structure often used for partitioned parquet files for example /year=2020/month=4/day=24/data.paquet

amishra-dev · 2020-04-24T07:08:36Z

I will ask my python expert on this in a bit, but how about url encoding individual pieces of the path, is that an option?

sebastiaanzaaijer · 2020-04-24T08:06:59Z

I believe this will still leave the url encoded piece in the blob path. Actually I suspect that url encoding could be the source of the issue, as without providing an encoded url it does encode the / but not the = as you can see in the error details below. Perhaps the signature is generated for the path with different encoded characters than the request?

Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.
RequestId:<snip>
Time:2020-04-24T07:51:42.2702974Z
ErrorCode:AuthenticationFailed
Error:None
AuthenticationErrorDetail:The MAC signature found in the HTTP request '<snip> is not the same as any computed signature. Server used following string to sign: 'GET











x-ms-client-request-id:<snip>
x-ms-date:Fri, 24 Apr 2020 07:51:42 GMT
x-ms-range:bytes=0-33554431
x-ms-version:2019-07-07
/<snip:storage_account>/azcopytest/test=%2Ftest.txt'.
                                       ^ ^
                                       | |
                        not url encoded+ |
                                         + url endoded

xiafu-msft · 2020-04-28T07:18:12Z

Hi @sebastiaanzaaijer

I think it's a bug in yarl and we have created an issue in their repo
aio-libs/yarl#420

xiafu-msft · 2020-04-28T08:06:27Z

Hi @sebastiaanzaaijer
I accidentally found that a recent change in our sdk may coincidentally resolve this problem. (Probabaly yarl makes some decoding decision based on some rules.... I don't know)
I just tested download a blob named /test/test=/test.txt and it's working somehow. Though the change hasn't been released, but it should be available in the next release!
You are also welcome to use master branch to generate .whl file and install that before our release!

xiafu-msft · 2020-05-14T18:24:23Z

Hi @sebastiaanzaaijer

Would you like to confirm if you still have the problem?

Thanks

sebastiaanzaaijer · 2020-05-15T21:27:39Z

Hi,

I just upgraded to azure-storage-blob==12.3.1 using pip and I still appear to have the issue. I also installed azure-core, azure-common and azure-storage-blob from master but that also didn't seem to help.

blokje · 2020-08-13T20:06:13Z

It seems that this problem is only occuring in the aio package. When using the async version of the upload I get the following error:

RequestId:1b76f85b-201e-005c-74ad-715f44000000
Time:2020-08-13T20:04:59.1562038Z
ErrorCode:AuthenticationFailed
Error:None
AuthenticationErrorDetail:The MAC signature found in the HTTP request 'OfTv85pT0baHDy0kmJc1D9to98fMgSoGyTCy5f4R8aQ=' is not the same as any computed signature. Server used following string to sign: 'PUT


7592193

application/octet-stream



*


x-ms-blob-type:BlockBlob
x-ms-client-request-id:43dc5160-dda0-11ea-9c9b-a683e72c7779
x-ms-date:Thu, 13 Aug 2020 20:04:59 GMT
x-ms-version:2019-12-12
/XX/account-XX/30f122680000010000011f29fe-Hypothetische%20Marokkaan%20_%20Zondag%20met%20Lubach%20(S11)-2FB3kDQc60M.opus
timeout:30'.```

When using the synchronous packages this file uploads without a hick.

aviramha · 2020-08-26T19:09:01Z

Hi!
We're affected by this aswell.

xiafu-msft · 2020-08-26T19:12:44Z

Hi @aviramha and @blokje

We will take a look if we can find a workaround! Sorry about this

hayesgb · 2020-08-26T19:25:12Z

Seeing this as an issue on my end too. Exclusive to the async libraries and doesn't seem to be replicated with azurite.

aviramha · 2020-08-26T19:59:05Z

Hi, It seems I found the bug and the solution - I'll send a PR in few moments.

xiafu-msft · 2020-08-27T02:14:21Z

Hi @aviramha
Thanks so much for finding the workaround! I added a commit to populate this fix into other packages.
Just the CI failed somehow, it looks like not related to this change, while after we figure out what's going wrong with the CI we will update you.

aviramha · 2020-08-27T09:08:20Z

@xiafu-msft Anything I can do to speed up the process?
Also, What would be the ETA for availability on PyPI? I'm considering to run our fork for now..

xiafu-msft · 2020-08-27T17:36:08Z

Hi @aviramha

We can give you a private drop so that you can install the wheel file now, does this sound good?

aviramha · 2020-08-27T17:38:24Z

I already created a wheel in my fork. Thanks anyway!

xiafu-msft · 2020-09-01T17:16:18Z

The pr has been merged, it should be released soon #13355

aviramha · 2020-09-01T17:28:49Z

@xiafu-msft too bad it's a different PR and didn't get a co-author 😢

xiafu-msft · 2020-09-01T22:49:14Z

@aviramha sorry that I should have gave you a heads up that I didn't merge your pr, and thanks for the commit!
I didn't merge your PR because at that time the CI in your PR failed #13346, and I wasn't sure what caused the failure, plus I added a commit 3912ea6 to your PR and I was worried my change broke the CI, so I created a new PR to play around with. (If I use your PR to play around you will get a lot of notifications)
Finally it turned out CI failure was caused by the changes in our dependency, and the CI failure was blocking other people's pr, so once I got the approval I didn't think too much and merged the fix along with the copied commit.

I will revert the pr partially and use your pr #13501
Sorry to upset you, I just merged that in a rush and forgot to add you as a co-author

ghost added customer-reported Issues that are reported by GitHub users external to the Azure organization. question The issue doesn't require a change to the product in order to be resolved. Most issues start as that labels Apr 23, 2020

kaerm added Client This issue points to a problem in the data-plane of the library. Service Attention This issue is responsible by Azure service team. Storage Storage Service (Queues, Blobs, Files) labels Apr 23, 2020

xiafu-msft self-assigned this Apr 28, 2020

xiafu-msft added bug This issue requires a change to an existing behavior in the product in order to be resolved. and removed question The issue doesn't require a change to the product in order to be resolved. Most issues start as that labels Jul 13, 2020

lmazuel added this to the Backlog milestone Jul 14, 2020

tasherif-msft self-assigned this Aug 19, 2020

aviramha mentioned this issue Aug 26, 2020

async download_blob authenticationerror when equal sign '=' in the url #9783

Closed

aviramha mentioned this issue Aug 26, 2020

fix authorization header on asyncio requests containing url-encoded chars #13346

Merged

xiafu-msft mentioned this issue Sep 1, 2020

edit all authentication files and add a test #13355

Merged

xiafu-msft closed this as completed in #13346 Sep 2, 2020

github-actions bot locked and limited conversation to collaborators Apr 12, 2023

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[azure-storage-blob] Blob uploads and downloads give authentication error when certain characters are in the blob path #11028

[azure-storage-blob] Blob uploads and downloads give authentication error when certain characters are in the blob path #11028

sebastiaanzaaijer commented Apr 23, 2020 •

edited

ghost commented Apr 23, 2020

amishra-dev commented Apr 24, 2020

sebastiaanzaaijer commented Apr 24, 2020

amishra-dev commented Apr 24, 2020

sebastiaanzaaijer commented Apr 24, 2020

xiafu-msft commented Apr 28, 2020

xiafu-msft commented Apr 28, 2020 •

edited

xiafu-msft commented May 14, 2020

sebastiaanzaaijer commented May 15, 2020

blokje commented Aug 13, 2020

aviramha commented Aug 26, 2020

xiafu-msft commented Aug 26, 2020

hayesgb commented Aug 26, 2020 •

edited

aviramha commented Aug 26, 2020 •

edited

xiafu-msft commented Aug 27, 2020

aviramha commented Aug 27, 2020

xiafu-msft commented Aug 27, 2020

aviramha commented Aug 27, 2020

xiafu-msft commented Sep 1, 2020

aviramha commented Sep 1, 2020

xiafu-msft commented Sep 1, 2020 •

edited

[azure-storage-blob] Blob uploads and downloads give authentication error when certain characters are in the blob path #11028

[azure-storage-blob] Blob uploads and downloads give authentication error when certain characters are in the blob path #11028

Comments

sebastiaanzaaijer commented Apr 23, 2020 • edited

ghost commented Apr 23, 2020

amishra-dev commented Apr 24, 2020

sebastiaanzaaijer commented Apr 24, 2020

amishra-dev commented Apr 24, 2020

sebastiaanzaaijer commented Apr 24, 2020

xiafu-msft commented Apr 28, 2020

xiafu-msft commented Apr 28, 2020 • edited

xiafu-msft commented May 14, 2020

sebastiaanzaaijer commented May 15, 2020

blokje commented Aug 13, 2020

aviramha commented Aug 26, 2020

xiafu-msft commented Aug 26, 2020

hayesgb commented Aug 26, 2020 • edited

aviramha commented Aug 26, 2020 • edited

xiafu-msft commented Aug 27, 2020

aviramha commented Aug 27, 2020

xiafu-msft commented Aug 27, 2020

aviramha commented Aug 27, 2020

xiafu-msft commented Sep 1, 2020

aviramha commented Sep 1, 2020

xiafu-msft commented Sep 1, 2020 • edited

sebastiaanzaaijer commented Apr 23, 2020 •

edited

xiafu-msft commented Apr 28, 2020 •

edited

hayesgb commented Aug 26, 2020 •

edited

aviramha commented Aug 26, 2020 •

edited

xiafu-msft commented Sep 1, 2020 •

edited