![](/img/trans.png)
[英]copy data from on premise sql server to delta format in Azure Data Lake Storage Gen2
[英]Copy Azure Storage Table Service data to SQL Server
如何使用PowerShell從Azure存儲表服務復制海量數據到SQL服務器? 微軟推薦使用的AzTable不支持沒有分區鍵底層信息的增量加載,文檔網站已經宕機了……: https:https://learn.microsoft.com/en-us/azure/storage/tables/table-storage -如何使用-powershell
我自己回答這個問題,因為我在網上找到解決方案時遇到了很多麻煩,並且想幫助其他人解決這個問題。
$StorageAccountResourceGroup = "MyResourceGroup"
$StorageAccountName = "MyStorageAccount"
$TableName = "MyStorageTableName"
$SqlConnectionString = "MySqlConnectionString"
$BulkCopy = [System.Data.SqlClient.SqlBulkCopy]::new()
$BulkCopy.DestinationTableName = "[myschema].[mytablename]"
$DataTable = <<Generate datatable from SQL-table>> # Code not included
Connect-AzAccount
$StorageAccountKey = (Get-AzStorageAccountKey -ResourceGroupName $StorageAccountResourceGroup -AccountName $StorageAccountName | Where-Object -FilterScript {$_.KeyName -eq "Key1"}).Value
$Context = New-AzStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $StorageAccountKey
$CloudTable = (Get-AzStorageTable -Context $Context -Name $TableName).CloudTable
$TableQuery = [Microsoft.Azure.Cosmos.Table.TableQuery]::new()
$Token = $null
BulkCounter = 0
do{
$ReturnObject = $CloudTable.ExecuteQuerySegmented($TableQuery, $Token)
$Token = $ReturnObject.ContinuationToken
foreach($Entry in $ReturnObject){
$Row = $DataTable.NewRow()
foreach($Column in $DataTable.Columns){
if($Column.ColumnName -eq "TimeStamp"){
$Value = $Entry.TimeStamp
} elseif($Column.ColumnName -eq "RowKey"){
$Value = $Entry.RowKey
} elseif($Column.ColumnName -eq "PartitionKey"){
$Value = $Entry.PartitionKey
} else {
if($Column.DataType -eq [System.Decimal]){
$Value = $Entry.Properties.$($Column.ColumnName).DoubleValue
} elseif($Column.DataType -eq [System.String]){
$Value = $Entry.Properties.$($Column.ColumnName).StringValue
} elseif($Column.DataType -eq [System.Guid]){
$Value = $Entry.Properties.$($Column.ColumnName).GuidValue
} elseif($Column.DataType -eq [System.datetimeoffset]){
$Value = $Entry.Properties.$($Column.ColumnName).DateTimeOffsetValue
} elseif($Column.DataType -eq [System.Int32]){
$Value = $Entry.Properties.$($Column.ColumnName).Int32Value
} elseif($Column.DataType -eq [System.Int64]){
$Value = $Entry.Properties.$($Column.ColumnName).Int64Value
} elseif($Column.DataType -eq [System.Boolean]){
$Value = $Entry.Properties.$($Column.ColumnName).BooleanValue
} elseif($Column.DataType -eq [System.Binary]){
$Value = $Entry.Properties.$($Column.ColumnName).BinaryValue
}
}
if([System.String]::IsNullOrWhiteSpace($Value)){
$Value = [System.DBNull]::value
}
$Row.($Column.ColumnName) = $Value
}
$DataTable.Rows.Add($Row)
$Counter++
}
$BulkCounter ++
if($BulkCounter % 25 -eq 0 -and $BulkCounter -ne 0){
$BulkCopy.WriteToServer($Datatable.CreateDataReader()) | Out-Null
$Datatable.Clear() | Out-Null
$BulkCounter = 0
}
} while ($Token)
if($Datatable.rows.count -gt 0){
$BulkCopy.WriteToServer($Datatable.CreateDataReader()) | Out-Null
$Datatable.Clear() | Out-Null
}
這里還有一些從存儲帳戶表自動生成表的函數:
function ConvertTo-SqlTypeFromEDM {
param (
$EDMType
)
if($EDMType -eq "String"){
return "varchar(100)"
} elseif($EDMType -eq "Guid"){
return "uniqueidentifier"
} elseif ($EDMType -eq "DateTime"){
return "datetimeoffset(0)"
} elseif ($EDMType -eq "Binary"){
return "varbinary(max)"
} elseif ($EDMType -eq "Int32"){
return "int"
} elseif ($EDMType -eq "Int64"){
return "long"
} elseif ($EDMType -eq "Double"){
return "decimal(25,5)"
} elseif ($EDMType -eq "Boolean"){
return "bit"
} else {
throw "Tybe $EDMType not implemented"
}
}
function Get-SqlQueryFromStorageTableSerivce {
param(
$CloudTable,
$SchemaName
)
$Query = [Microsoft.Azure.Cosmos.Table.TableQuery]::new()
$Query.TakeCount = 1
$Token = $null
$Result = $CloudTable.ExecuteQuerySegmented($Query, $Token)
$TableName = $CloudTable.Name
$TableString = "DROP TABLE IF EXISTS [$SchemaName].[$TableName]`n"
$TableString += "CREATE TABLE [$SchemaName].[$TableName] (`n`t"
$ColumnString = @()
$TableKeys = @()
if($Result.PartitionKey){
$ColumnString += "[PartitionKey] varchar(100) NOT NULL"
$TableKeys += "[PartitionKey]"
}
if($Result.RowKey){
$ColumnString += "[RowKey] varchar(100) NOT NULL"
$TableKeys += "[RowKey]"
}
if($Result.Timestamp){
$ColumnString += "[Timestamp] datetimeoffset(0) NULL"
}
foreach($Column in $Result.Properties.Keys){
$ColumnString += "[$Column] $(ConvertTo-SqlTypeFromEDM -EDMType $Result.Properties.$Column.PropertyType) NULL"
}
$TableString += $ColumnString -join ",`n`t"
$TableString += "`nCONSTRAINT [PK_1_$($TableName)_1] PRIMARY KEY CLUSTERED`n"
$TableString += "(`n`t" + ($TableKeys -join ",`n`t")
$TableString += "`n)WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF, DATA_COMPRESSION = PAGE) ON [PRIMARY]"
$TableString += "`n) ON [PRIMARY]`n"
return $TableString
}
聲明:本站的技術帖子網頁,遵循CC BY-SA 4.0協議,如果您需要轉載,請注明本站網址或者原文地址。任何問題請咨詢:yoyou2525@163.com.