[英]How to upload a file to ADLS Gen2 using Python
我正在尝试将文件从我的本地代码存储库上传到 adls:
import requests
import json
def auth(tenant_id, client_id, client_secret):
print('auth')
auth_headers = {
"Content-Type": "application/x-www-form-urlencoded"
}
auth_body = {
"client_id": client_id,
"client_secret": client_secret,
"scope" : "https://storage.azure.com/.default",
"grant_type" : "client_credentials"
}
resp = requests.post(f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token", headers=auth_headers, data=auth_body)
return (resp.status_code, json.loads(resp.text))
def mkfs(account_name, fs_name, access_token):
print('mkfs')
fs_headers = {
"Authorization": f"Bearer {access_token}"
}
resp = requests.put(f"https://{account_name}.dfs.core.windows.net/{fs_name}?resource=filesystem", headers=fs_headers)
return (resp.status_code, resp.text)
def mkdir(account_name, fs_name, dir_name, access_token):
print('mkdir')
dir_headers = {
"Authorization": f"Bearer {access_token}"
}
resp = requests.put(f"https://{account_name}.dfs.core.windows.net/{fs_name}/{dir_name}?resource=directory", headers=dir_headers)
return (resp.status_code, resp.text)
def touch_file(account_name, fs_name, dir_name, file_name, access_token):
print('touch_file')
touch_file_headers = {
"Authorization": f"Bearer {access_token}"
}
resp = requests.put(f"https://{account_name}.dfs.core.windows.net/{fs_name}/{dir_name}/{file_name}?resource=file", headers=touch_file_headers)
return (resp.status_code, resp.text)
def append_file(account_name, fs_name, path, content, position, access_token):
print('append_file')
append_file_headers = {
"Authorization": f"Bearer {access_token}",
"Content-Type": "text/plain",
"Content-Length": f"{len(content)}"
}
resp = requests.patch(f"https://{account_name}.dfs.core.windows.net/{fs_name}/{path}?action=append&position={position}", headers=append_file_headers, data=content)
return (resp.status_code, resp.text)
def flush_file(account_name, fs_name, path, position, access_token):
print('flush_file')
flush_file_headers = {
"Authorization": f"Bearer {access_token}"
}
resp = requests.patch(f"https://{account_name}.dfs.core.windows.net/{fs_name}/{path}?action=flush&position={position}", headers=flush_file_headers)
return (resp.status_code, resp.text)
def mkfile(account_name, fs_name, dir_name, file_name, local_file_name, access_token):
print('mkfile')
status_code, result = touch_file(account_name, fs_name, dir_name, file_name, access_token)
if status_code == 201:
with open(local_file_name, 'rb') as local_file:
path = f"{dir_name}/{file_name}"
content = local_file.read()
position = 0
append_file(account_name, fs_name, path, content, position, access_token)
position = len(content)
flush_file(account_name, fs_name, path, position, access_token)
else:
print(result)
if __name__ == '__main__':
tenant_id = <tenant_id>
client_id = <client_id>
client_secret = <client_secret>
account_name = 'adlsprod'
fs_name = 'fs-adls'
dir_name = 'FileStore'
file_name = "abc.py"
local_file_name = "__main__.py"
# Acquire an Access token
auth_status_code, auth_result = auth(tenant_id, client_id, client_secret)
access_token = auth_status_code == 200 and auth_result['access_token'] or ''
print(access_token)
# Create a filesystem
mkfs_status_code, mkfs_result = mkfs(account_name, fs_name, access_token)
print(mkfs_status_code, mkfs_result)
# Create a directory
mkdir_status_code, mkdir_result = mkdir(account_name, fs_name, dir_name, access_token)
print(mkdir_status_code, mkdir_result)
# Create a file from local file
mkfile(account_name, fs_name, dir_name, file_name, local_file_name, access_token)
但我收到以下错误:
mkfs
403 {"error":{"code":"AuthorizationFailure","message":"This request is not authorized to perform this operation.\nRequestId:183befbd-f01f-00f7-562e-d8a0b8000000\nTime:2022-10-04T20:19:34.5637539Z"}}
我认为 fs_name 和 account_name 在 url 中的位置不正确。
这是 adls 中示例文件的位置 -
abfss://fs-adls@adlsprod.dfs.core.windows.net/FileStore/abc.txt
我从这个 url 中选择了 account_name 和 fs_name。我不明白我哪里出错了,或者有没有其他方法可以使用 python 将文件上传到 adls 文件夹。
我确定您可以使用常规请求包来执行此操作,但是 Azure Python SDK 使这一切变得更加容易: https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-存储目录文件-acl-python
在示例中:
def upload_file_to_directory():
try:
file_system_client = service_client.get_file_system_client(file_system="my-file-system")
directory_client = file_system_client.get_directory_client("my-directory")
file_client = directory_client.create_file("uploaded-file.txt")
local_file = open("C:\\file-to-upload.txt",'r')
file_contents = local_file.read()
file_client.append_data(data=file_contents, offset=0, length=len(file_contents))
file_client.flush_data(len(file_contents))
except Exception as e:
print(e)
您可以在帐户密钥(看起来像您正在做的)或 Azure Active Directory(在本地计算机上可能更容易)之间进行选择
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.