Append block in blob service python

Question

Azure blob storage service with Append Block in python azure module doesn't work. When i tried to modify the module it shows the below error.

Traceback (most recent call last):
  File "C:\Program Files (x86)\Microsoft Visual Studio 12.0\Common7\IDE\Extensions\Microsoft\Python Tools for Visual Studio\2.1\visualstudio_py_util.py", line 106, in exec_file
    exec_code(code, file, global_variables)
  File "C:\Program Files (x86)\Microsoft Visual Studio 12.0\Common7\IDE\Extensions\Microsoft\Python Tools for Visual Studio\2.1\visualstudio_py_util.py", line 82, in exec_code
    exec(code_obj, global_variables)
  File "E:\Finons\Sankar\Source\Source\WebsiteTracking\testing.py", line 15, in<module>
  blob_service.append_block_blob_from_file(containername,filename,filepath,None)
  File "C:\Python27\lib\site-packages\azure\storage\blobservice.py", line 2723, in append_block_blob_from_file
  x_ms_lease_id,
  File "C:\Python27\lib\site-packages\azure\storage\blobservice.py", line 851, in put_blob    self._perform_request(request)
  File "C:\Python27\lib\site-packages\azure\storage\storageclient.py", line 179,  in _perform_request
    _storage_error_handler(ex)
  File "C:\Python27\lib\site-packages\azure\storage\__init__.py", line 1186, in _storage_error_handler
    return _general_error_handler(http_error)
  File "C:\Python27\lib\site-packages\azure\__init__.py", line 551, in _general_error_handler
    http_error.respbody.decode('utf-8-sig'))
WindowsAzureError: Unknown error (Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.)
<?xml version="1.0" encoding="utf-8"?><Error><Code>AuthenticationFailed</Code><Message>Server failed to authenticate the request. Make sure the value of Authorization header is formed correctly including the signature.
RequestId:898a6c60-0001-0094-2b99-d6da0d000000 Time:2015-08-14T13:58:54.9779810Z</Message><AuthenticationErrorDetail>The MAC signature found in the HTTP request 'DAAfD/aLQHqljUS35p7CoX+JBc5lyrPr1twQIQEW0HI='
 is not the same as any computed signature. Server used following string to sign: 'PUT

application/octet-stream Charset=UTF-8

Is there any possibilities to resolve this error?

This is my code changes in the module given by SDK.

def append_block_blob_from_file(self, container_name, blob_name, stream,
                             count=None, content_encoding=None,
                             content_language=None, content_md5=None,
                             cache_control=None,
                             x_ms_blob_content_type=None,
                             x_ms_blob_content_encoding=None,
                             x_ms_blob_content_language=None,
                             x_ms_blob_content_md5=None,
                             x_ms_blob_cache_control=None,
                             x_ms_meta_name_values=None,
                             x_ms_lease_id=None, progress_callback=None,
                             max_connections=1, max_retries=5, retry_wait=1.0):

    _validate_not_none('container_name', container_name)
    _validate_not_none('blob_name', blob_name)
    _validate_not_none('stream', stream)

    if count and count < self._BLOB_MAX_DATA_SIZE:
        if progress_callback:
            progress_callback(0, count)

        data = stream.read(count)
        self.put_blob(container_name,
                      blob_name,
                      data,
                      'AppendBlob',
                      content_encoding,
                      content_language,
                      content_md5,
                      cache_control,
                      x_ms_blob_content_type,
                      x_ms_blob_content_encoding,
                      x_ms_blob_content_language,
                      x_ms_blob_content_md5,
                      x_ms_blob_cache_control,
                      x_ms_meta_name_values,
                      x_ms_lease_id)

        if progress_callback:
            progress_callback(count, count)
    else:
        self.put_blob(
            container_name,
            blob_name,
            None,
            'AppendBlob',
            content_encoding,
            content_language,
            content_md5,
            cache_control,
            x_ms_blob_content_type,
            x_ms_blob_content_encoding,
            x_ms_blob_content_language,
            x_ms_blob_content_md5,
            x_ms_blob_cache_control,
            x_ms_meta_name_values,
            x_ms_lease_id,
        )

        _upload_blob_chunks(
            self,
            container_name,
            blob_name,
            count,
            self._BLOB_MAX_CHUNK_DATA_SIZE,
            stream,
            max_connections,
            max_retries,
            retry_wait,
            progress_callback,
            x_ms_lease_id,
            _AppendBlobChunkUploader,
        )

        self.put_block_list(
            container_name,
            blob_name,
            block_ids,
            content_md5,
            x_ms_blob_cache_control,
            x_ms_blob_content_type,
            x_ms_blob_content_encoding,
            x_ms_blob_content_language,
            x_ms_blob_content_md5,
            x_ms_meta_name_values,
            x_ms_lease_id,
        )

and i defined the below code in Init.py

class _AppendBlobChunkUploader(_BlobChunkUploader):
def _append_chunk(self, chunk_offset, chunk_data):
    range_id = url_quote(_encode_base64('{0:032d}'.format(chunk_offset)))
    self.blob_service.put_block(
        self.container_name,
        self.blob_name,
        chunk_data,
        range_id,
        x_ms_lease_id=self.x_ms_lease_id,x_ms_blob_condition_maxsize=4194304,x_ms_blob_condition_appendpos=chunk_offset
    )
    return range_id

Please Note two additional parameters at the end. I included this after referring the api reference. Also modified the version as X_MS_VERSION = '2015-02-21'

Ref Link:https://msdn.microsoft.com/en-us/library/azure/mt427365.aspx

I looked up source code for Azure SDK for Python here: github.com/Azure/azure-sdk-for-python/blob/master/azure/storage/… and I could not find append_block_blob_from_file function in the SDK. Can you tell us from which SDK are you using? — Gaurav Mantri
– Gaurav Mantri, Commented Aug 14, 2015 at 14:09
As i said earlier, i modified the python module and created this append_block_blob_from_file function just like put_block_blob_from_file. I Included all the needed changes in init.py and the other supporting files. — Sankar
– Sankar, Commented Aug 14, 2015 at 14:19
Please find my code changes in the module, updated question. — Sankar
– Sankar, Commented Aug 14, 2015 at 14:40
I suspect some changes in signature generation procedure please help. — Sankar
– Sankar, Commented Aug 14, 2015 at 14:43

Will Shao - MSFT · Accepted Answer · 2015-08-19 01:58:28Z

It seems that you didn't provide the put_block function details from your description. I checked the Python SDK, I didn't find the put_block function supports those two parameters: x_ms_blob_condition_maxsize and x_ms_blob_condition_appendpos in BlobService class. However, for your scenarios, I suggest you can use Azure Blob REST API to append your block as Gaurav's said. At the same time, from your error message, I am sure your signature has something wrong. I have use a simple code to construct the signature string, please refer to it:

import urllib
import hashlib
import hmac
import base64
import urllib.request
from wsgiref.handlers import format_date_time
from datetime import datetime
from time import mktime
account_name='****'
account_key='*****'
account_container='***'
file_name='**'
file_path='**'
request_url='https://myaccount.blob.core.windows.net/mycontainer/myblob?comp=appendblock' 
#https://myaccount.blob.core.windows.net/mycontainer/myblob?comp=appendblock  Put
blobLength=0
blockID='**'
#create the authorization header
#how to encode and decode signtostring 
def createAuthorizationHeader(canonicalizedString):
    storage_account_key = base64.b64decode(account_key)
    byte_canonicalizedString=canonicalizedString.encode('utf-8')
    signature = base64.b64encode(hmac.new(key=storage_account_key,msg=byte_canonicalizedString, digestmod=hashlib.sha256).digest())
    print(signature)
    authorization_str="SharedKey "+account_name+":"+signature.decode('utf-8')
    print(authorization_str)
    return authorization_str


#Construction Sign
def constructSTS(http_verb,Content_Encoding='',Content_Language='',Content_MD5='',Content_Type='',Date='',
                 If_Modified_Since='',If_Match='',If_None_Match='',length='',
                 If_Unmodified_Since='',Range='',CanonicalizedHeaders='',CanonicalizedResource=''):
    StringToSign=http_verb+'\n'+Content_Encoding+'\n'+Content_Language+'\n'+Content_MD5+'\n'+Content_Type+'\n'+Date+'\n'+If_Modified_Since+'\n'+If_Match+'\n'+If_None_Match+'\n'+length+'\n'+If_Unmodified_Since+'\n'+Range+'\n'+CanonicalizedHeaders+CanonicalizedResource;
    print(StringToSign)
    return StringToSign


#Construction Canonicalized headers
def constructCanonicalizedHeaders(date):    
    CanonicalizedHeader="x-ms-date:"+date+"\nx-ms-version:2015-02-21\nx-ms-lease-id:"+blockID+"\nms-blob-condition-appendpos:"+youroffset+"\nx-ms-blob-condition-maxsize:4194304\n"
    print(CanonicalizedHeader)
    return CanonicalizedHeader

#Construction  Canonicalized Resource
def constructCanonicalizedResource():
    canonicalizedResource="/"+account_name+"/"+account_container+"/"+file_name+"\ncomp:appendblock"
    print(canonicalizedResource)
    return canonicalizedResource

#get current date
def getCurrentDate():
    now = datetime.now()
    stamp = mktime(now.timetuple())
    date=format_date_time(stamp)
    return date
#Create http request
def createHttpRequest():
    date=getCurrentDate()
    r=urllib.request.Request(request_url)    
    canonicalizedString=constructSTS("PUT",CanonicalizedHeaders=constructCanonicalizedHeaders(date),CanonicalizedResource=constructCanonicalizedResource(),length=blobLength)
    print(canonicalizedString)
    r.add_header('x-ms-version','2015-02-21');
    r.add_header('Authorization',createAuthorizationHeader(canonicalizedString))
    r.add_header('x-ms-date',date)
    r.add_header('Content-Length',blobLength)
    r.add_header('x-ms-lease-id',blockID)
    r.add_header("x-ms-blob-condition-appendpos",youroffset)
    r.add_header("x-ms-blob-condition-maxsize",4194304)

You should get your blockblob length and send blockblob data to server using this request.

cipri.l · Accepted Answer · 2019-03-14 17:35:27Z

2

just in case someone else comes across the same issue, here is a link with an example how to append from python

from azure.storage.blob import AppendBlobService

append_blob_service = AppendBlobService(account_name='myaccount', account_key='mykey')

# The same containers can hold all types of blobs
append_blob_service.create_container('mycontainer')

# Append blobs must be created before they are appended to
append_blob_service.create_blob('mycontainer', 'myappendblob')
append_blob_service.append_blob_from_text('mycontainer', 'myappendblob', u'Hello, world!')

append_blob = append_blob_service.get_blob_to_text('mycontainer', 'myappendblob')

answered Mar 14, 2019 at 17:35

cipri.l

81911 silver badges23 bronze badges

1 Comment

Tobias Over a year ago

The SDK also contains some samples: github.com/Azure/azure-storage-python/blob/master/samples/blob/…

Collectives™ on Stack Overflow

Append block in blob service python

2 Answers 2

Comments

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related