package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)
// 定義全域變數
var (
region string // 儲存地區
bucketName string // 儲存空間名稱
prefix string // 對象首碼（檔案夾路徑）
localDir string // 本地下載目錄
maxWorkers int // 最大並發數
maxKeys int // 每次列舉的最大對象數
)
// DownloadTask 下載任務結構
type DownloadTask struct {
ObjectKey string
LocalPath string
Size int64
}
// DownloadResult 下載結果結構
type DownloadResult struct {
ObjectKey string
Success bool
Error error
Size int64
}
// init函數用於初始化命令列參數
func init() {
flag.StringVar(®ion, "region", "", "The region in which the bucket is located.")
flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
flag.StringVar(&prefix, "prefix", "", "The prefix (folder path) to download.")
flag.StringVar(&localDir, "local-dir", "./downloads", "Local directory to save downloaded files.")
flag.IntVar(&maxWorkers, "workers", 5, "Maximum number of concurrent downloads.")
flag.IntVar(&maxKeys, "max-keys", 1000, "Maximum number of objects to list at once.")
}
func main() {
// 解析命令列參數
flag.Parse()
// 檢查必要參數
if len(bucketName) == 0 {
flag.PrintDefaults()
log.Fatalf("invalid parameters, bucket name required")
}
if len(region) == 0 {
flag.PrintDefaults()
log.Fatalf("invalid parameters, region required")
}
// 確保首碼以/結尾（如果不是Null 字元串）
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
// 建立本地下載目錄
if err := os.MkdirAll(localDir, 0755); err != nil {
log.Fatalf("failed to create local directory: %v", err)
}
// 載入預設配置並設定憑證提供者和地區
cfg := oss.LoadDefaultConfig().
WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
WithRegion(region)
// 建立OSS用戶端
client := oss.NewClient(cfg)
fmt.Printf("開始批量下載，儲存空間: %s, 首碼: %s, 本地目錄: %s\n", bucketName, prefix, localDir)
// 列舉所有需要下載的對象
tasks, err := listObjects(client, bucketName, prefix)
if err != nil {
log.Fatalf("failed to list objects: %v", err)
}
if len(tasks) == 0 {
fmt.Println("沒有找到需要下載的檔案")
return
}
fmt.Printf("找到 %d 個檔案需要下載\n", len(tasks))
// 執行批量下載
results := batchDownload(client, tasks, maxWorkers)
// 統計下載結果
var successCount, failCount int
var totalSize int64
for _, result := range results {
if result.Success {
successCount++
totalSize += result.Size
} else {
failCount++
fmt.Printf("下載失敗: %s, 錯誤: %v\n", result.ObjectKey, result.Error)
}
}
fmt.Printf("\n下載完成! 成功: %d, 失敗: %d, 總大小: %s\n",
successCount, failCount, formatBytes(totalSize))
}
// listObjects 列舉儲存空間中指定首碼的所有對象
func listObjects(client *oss.Client, bucketName, prefix string) ([]DownloadTask, error) {
var tasks []DownloadTask
var continuationToken *string
for {
// 建立列舉對象請求
request := &oss.ListObjectsV2Request{
Bucket: oss.Ptr(bucketName),
Prefix: oss.Ptr(prefix),
MaxKeys: int32(maxKeys),
ContinuationToken: continuationToken,
}
// 執行列舉操作
result, err := client.ListObjectsV2(context.TODO(), request)
if err != nil {
return nil, fmt.Errorf("failed to list objects: %w", err)
}
// 處理列舉結果
for _, obj := range result.Contents {
// 跳過檔案夾對象（以/結尾且大小為0）
if strings.HasSuffix(*obj.Key, "/") && obj.Size == 0 {
continue
}
// 計算本地檔案路徑
relativePath := strings.TrimPrefix(*obj.Key, prefix)
localPath := filepath.Join(localDir, relativePath)
tasks = append(tasks, DownloadTask{
ObjectKey: *obj.Key,
LocalPath: localPath,
Size: obj.Size,
})
}
// 檢查是否還有更多個物件
if result.NextContinuationToken == nil {
break
}
continuationToken = result.NextContinuationToken
}
return tasks, nil
}
// batchDownload 執行批量下載
func batchDownload(client *oss.Client, tasks []DownloadTask, maxWorkers int) []DownloadResult {
taskChan := make(chan DownloadTask, len(tasks))
resultChan := make(chan DownloadResult, len(tasks))
// 啟動工作協程
var wg sync.WaitGroup
for i := 0; i < maxWorkers; i++ {
wg.Add(1)
go downloadWorker(client, bucketName, taskChan, resultChan, &wg)
}
// 發送下載任務
go func() {
for _, task := range tasks {
taskChan <- task
}
close(taskChan)
}()
// 等待所有工作協程完成
go func() {
wg.Wait()
close(resultChan)
}()
// 收集結果並顯示進度
var results []DownloadResult
completed := 0
total := len(tasks)
for result := range resultChan {
results = append(results, result)
completed++
if result.Success {
fmt.Printf("✓ [%d/%d] %s (%s)\n",
completed, total, result.ObjectKey, formatBytes(result.Size))
} else {
fmt.Printf("✗ [%d/%d] %s - 錯誤: %v\n",
completed, total, result.ObjectKey, result.Error)
}
}
return results
}
// downloadWorker 下載工作協程
func downloadWorker(client *oss.Client, bucketName string, taskChan <-chan DownloadTask,
resultChan chan<- DownloadResult, wg *sync.WaitGroup) {
defer wg.Done()
for task := range taskChan {
result := DownloadResult{
ObjectKey: task.ObjectKey,
Size: task.Size,
}
// 建立本地檔案目錄
if err := os.MkdirAll(filepath.Dir(task.LocalPath), 0755); err != nil {
result.Error = fmt.Errorf("failed to create directory: %w", err)
resultChan <- result
continue
}
// 檢查檔案是否已存在且大小一致
if fileInfo, err := os.Stat(task.LocalPath); err == nil {
if fileInfo.Size() == task.Size {
result.Success = true
resultChan <- result
continue
}
}
// 建立下載請求
getRequest := &oss.GetObjectRequest{
Bucket: oss.Ptr(bucketName),
Key: oss.Ptr(task.ObjectKey),
}
// 執行下載
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
_, err := client.GetObjectToFile(ctx, getRequest, task.LocalPath)
cancel()
if err != nil {
result.Error = err
} else {
result.Success = true
}
resultChan <- result
}
}
// formatBytes 格式化位元組數為可讀格式
func formatBytes(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
使用方法
# 編譯測試代碼為可執行檔
go build -o oss-batch-download main.go
# 下載指定檔案夾
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix images/2024/
# 自訂本地目錄和並發數
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix documents/ -local-dir ./downloads -workers 10
# 下載整個儲存空間
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix ""
輸出樣本
程式運行時會顯示詳細的下載進度：
開始批量下載，儲存空間: my-bucket, 首碼: images/2024/, 本地目錄: ./downloads
找到 150 個檔案需要下載
✓ [1/150] images/2024/photo1.jpg (2.3 MB)
✓ [2/150] images/2024/photo2.png (1.8 MB)
...
下載完成! 成功: 148, 失敗: 2, 總大小: 1.2 GB