Upload local folder to Azure Blob Storage using BlobServiceClient with Python V12 SDK

Question

Summarize the problem:

I am trying to upload a local folder to Blob Storage using BlobServiceClient with Python. Some of the questions here and here do not work because create_blob_from_path() doesn't work in V12 SDK and I wouldn't want to go back to older version.

What I've tried:

I am using os.walk for local directory but missing the most important part like a function similar to create_blob_from_path() .

Sample code:

from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, PublicAccess
import os 

base_file_path = '/path/to/my/local/directory/'
connect_str = '1q2w3e4r5t6y'
container_name = 'abc'

try: 
    blob_service_client = BlobServiceClient.from_connection_string(connect_str)
    container_name = 'abc' # already created in Azure 
    container_client = blob_service_client.get_container_client(container_name)
   
    upload_local_file_path = base_file_path + 'csv-summary-output' # input folder path

    for root, subdir, local_file in os.walk(upload_local_file_path):
        if local_file:
            for name in local_file:
                dir_part = os.path.relpath(root, upload_local_file_path)
                file_path = os.path.join(root, name)
                ==> missing parts here
except Exception as ex:
    print('Exception:')
    print(ex)

Any help is much appreciated and I will take a look at Azure Github to see if anything useful there.

Answer 1

You can also use the code below(Assume the local folder is in D:\aaa , please feel free to modify the code as per your need):

from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient,PublicAccess
import os

def run_sample():    
    conn_str="xxx"
    container_name="xxx"    
    
    path_remove = "D:\\"
    local_path = "D:\\aaa" #the local folder

    service_client=BlobServiceClient.from_connection_string(conn_str)
    container_client = service_client.get_container_client(container_name)  

    for r,d,f in os.walk(local_path):
        if f:
            for file in f:
                file_path_on_azure = os.path.join(r,file).replace(path_remove,"")
                file_path_on_local = os.path.join(r,file)

                blob_client = container_client.get_blob_client(file_path_on_azure)

                with open(file_path_on_local,'rb') as data:
                    blob_client.upload_blob(data)


if __name__ == '__main__':
    run_sample()
    print("**completed**")

Answer 2

OK. With the help of source code from Git , I was able to figure out the solution and I am posting here for future references. I was very confused about dest variable and was even looking for container's url to give an upload path. It's actually been taken care of in upload_dir function. Any other suggestions are also welcomed.

Sample code:

from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, PublicAccess
import os 

base_file_path = '/path/to/your/local/directory/'
# target_folder is the subfolder under container_name 
target_folder = 'xyz' 
 
connect_str = '1q2w3e4r5t6y7u8i9o0p'
container_name = 'abc'

def upload_file(source, dest):
    
    print(f'Uploading {source} to {dest}')
    with open(source, 'rb') as data:
      client.upload_blob(name=dest, data=data)

def upload_dir(source, dest):

    prefix = '' if dest == '' else dest + '/'
    prefix += os.path.basename(source) + '/'
    for root, dirs, files in os.walk(source):
        for name in files:
            dir_part = os.path.relpath(root, source)
            dir_part = '' if dir_part == '.' else dir_part + '/'
            file_path = os.path.join(root, name)
            blob_path = prefix + dir_part + name
            upload_file(file_path, blob_path)
try:
    source = base_file_path + target_folder
    dest = '' # dest is the target folder name  
    service_client = BlobServiceClient.from_connection_string(connect_str)
    client = service_client.get_container_client(container_name)
except Exception as ex:
    print('Exception:')
    print(ex)

if __name__ == '__main__':
    upload_dir(source=source, dest=dest)