Object Storage Service:Download objects to a local device using OSS SDK for Go 2.0

package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables.
var (
	region     string // The region in which the bucket is located.
	bucketName string // The name of the bucket.
	objectName string // The name of the object.
)

// Use the init function to initialize parameters.
func init() {
	flag.StringVar(&region, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&objectName, "object", "", "The name of the object.")
}

func main() {
	// Parse parameters.
	flag.Parse()

	// Check whether the bucket name is empty.
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	// Check whether the region is empty.
	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Check whether the object name is empty.
	if len(objectName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, object name required")
	}

	// Load the default configurations and specify the credential provider and region.
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client.
	client := oss.NewClient(cfg)

	// Specify the path of the local file.
	localFile := "download.file"

	// For example, an object was last modified at 18:43:02, November 21, 2023. If the time specified in the IfModifiedSince condition is earlier than the last modified time, the object is downloaded. 
	date := time.Date(2024, time.October, 21, 18, 43, 2, 0, time.UTC)

	// Assume that the ETag of an object is e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855. If the ETag specified in the IfMatch condition matches the ETag of the object, the object is downloaded. 
	etag := "\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\""

	// Create a request to download the object.
	getRequest := &oss.GetObjectRequest{
		Bucket:          oss.Ptr(bucketName),                   // The name of the bucket.
		Key:             oss.Ptr(objectName),                   // The name of the object.
		IfModifiedSince: oss.Ptr(date.Format(http.TimeFormat)), // Specify the IfModifiedSince condition.
		IfMatch:         oss.Ptr(etag),                         // Specify the IfMatch condition.
	}

	// Download the object to a local device and process the result.
	result, err := client.GetObjectToFile(context.TODO(), getRequest, localFile)
	if err != nil {
		log.Fatalf("failed to get object to file %v", err)
	}

	log.Printf("get object to file result:%#v\n", result)
}

package main

import (
	"context"
	"flag"
	"fmt"
	"log"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables.
var (
	region     string // The region in which the bucket is located.
	bucketName string // The name of the bucket.
	objectName string // The name of the object.
)

// Use the init function to initialize parameters.
func init() {
	flag.StringVar(&region, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&objectName, "object", "", "The name of the object.")
}

func main() {
	// Parse parameters.
	flag.Parse()

	// Check whether the bucket name is empty.
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	// Check whether the region is empty.
	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Check whether the object name is empty.
	if len(objectName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, object name required")
	}

	// Load the default configurations and specify the credential provider and region.
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client.
	client := oss.NewClient(cfg)

	// Specify the path of the local file.
	localFile := "download.file"

	// Create a request to download the object.
	getRequest := &oss.GetObjectRequest{
		Bucket: oss.Ptr(bucketName), // The name of the bucket.
		Key:    oss.Ptr(objectName), // The name of the object.
		ProgressFn: func(increment, transferred, total int64) {
			fmt.Printf("increment:%v, transferred:%v, total:%v\n", increment, transferred, total)
		}, // Display the download progress.
	}

	// Download the object to a local device and process the result.
	result, err := client.GetObjectToFile(context.TODO(), getRequest, localFile)
	if err != nil {
		log.Fatalf("failed to get object to file %v", err)
	}

	log.Printf("get object to file result:%#v\n", result)
}

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables.
var (
	region     string // The region in which the bucket is located.
	bucketName string // The name of the bucket.
	prefix     string // The prefix (file path) of the object.
	localDir   string // The local directory.
	maxWorkers int    // The maximum concurrency.
	maxKeys    int    // The maximum number of objects that can be listed.
)

// The structure of the download task.
type DownloadTask struct {
	ObjectKey string
	LocalPath string
	Size      int64
}

// The structure of the download result.
type DownloadResult struct {
	ObjectKey string
	Success   bool
	Error     error
	Size      int64
}

// Define the init function used to initialize parameters.
func init() {
	flag.StringVar(&region, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&prefix, "prefix", "", "The prefix (folder path) to download.")
	flag.StringVar(&localDir, "local-dir", "./downloads", "Local directory to save downloaded files.")
	flag.IntVar(&maxWorkers, "workers", 5, "Maximum number of concurrent downloads.")
	flag.IntVar(&maxKeys, "max-keys", 1000, "Maximum number of objects to list at once.")
}

func main() {
	// Parse the command-line parameters.
	flag.Parse()

	// Check the required parameters.
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// To ensure a prefix ends with "/" if not empty.
	if prefix != "" && !strings.HasSuffix(prefix, "/") {
		prefix += "/"
	}

	// Create the local download directory.
	if err := os.MkdirAll(localDir, 0755); err != nil {
		log.Fatalf("failed to create local directory: %v", err)
	}

	// Load the default configurations and specify the credential provider and region.
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client.
	client := oss.NewClient(cfg)

	fmt.Printf("start batch download，bucket: %s, prefix: %s, local directory: %s\n", bucketName, prefix, localDir)

	// List all objects requiring download.
	tasks, err := listObjects(client, bucketName, prefix)
	if err != nil {
		log.Fatalf("failed to list objects: %v", err)
	}

	if len(tasks) == 0 {
		fmt.Println("No objects requiring download were found")
		return
	}

	fmt.Printf("%d objects need to be downloaded\n", len(tasks))

	// Execute batch download.
	results := batchDownload(client, tasks, maxWorkers)

	// Summarize the result of the download.
	var successCount, failCount int
	var totalSize int64
	for _, result := range results {
		if result.Success {
			successCount++
			totalSize += result.Size
		} else {
			failCount++
			fmt.Printf("Download failed: %s, Error: %v\n", result.ObjectKey, result.Error)
		}
	}

	fmt.Printf("\nDownload complete! success: %d, Failure: %d, Total size: %s\n",
		successCount, failCount, formatBytes(totalSize))
}

// List all objects in a bucket with a specified prefix.
func listObjects(client *oss.Client, bucketName, prefix string) ([]DownloadTask, error) {
	var tasks []DownloadTask
	var continuationToken *string

	for {
		// Create a request to list objects.
		request := &oss.ListObjectsV2Request{
			Bucket:            oss.Ptr(bucketName),
			Prefix:            oss.Ptr(prefix),
			MaxKeys:           int32(maxKeys),
			ContinuationToken: continuationToken,
		}

		// Execute the list operation.
		result, err := client.ListObjectsV2(context.TODO(), request)
		if err != nil {
			return nil, fmt.Errorf("failed to list objects: %w", err)
		}

		// Process the result of the list operation.
		for _, obj := range result.Contents {
			// Excluding folder objects ending with '/' and having zero size
			if strings.HasSuffix(*obj.Key, "/") && obj.Size == 0 {
				continue
			}

			// Computes the local file directory.
			relativePath := strings.TrimPrefix(*obj.Key, prefix)
			localPath := filepath.Join(localDir, relativePath)

			tasks = append(tasks, DownloadTask{
				ObjectKey: *obj.Key,
				LocalPath: localPath,
				Size:      obj.Size,
			})
		}

		// Check whether there are more objects.
		if result.NextContinuationToken == nil {
			break
		}
		continuationToken = result.NextContinuationToken
	}

	return tasks, nil
}

// Execute batch download operation.
func batchDownload(client *oss.Client, tasks []DownloadTask, maxWorkers int) []DownloadResult {
	taskChan := make(chan DownloadTask, len(tasks))
	resultChan := make(chan DownloadResult, len(tasks))

	// Launches the worker coroutine.
	var wg sync.WaitGroup
	for i := 0; i < maxWorkers; i++ {
		wg.Add(1)
		go downloadWorker(client, bucketName, taskChan, resultChan, &wg)
	}

	// Dispatch the download task.
	go func() {
		for _, task := range tasks {
			taskChan <- task
		}
		close(taskChan)
	}()

	// Awaits the completion of all worker coroutines.
	go func() {
		wg.Wait()
		close(resultChan)
	}()

	// Collect the results and display the progres.
	var results []DownloadResult
	completed := 0
	total := len(tasks)

	for result := range resultChan {
		results = append(results, result)
		completed++

		if result.Success {
			fmt.Printf("✓ [%d/%d] %s (%s)\n",
				completed, total, result.ObjectKey, formatBytes(result.Size))
		} else {
			fmt.Printf("✗ [%d/%d] %s - Error: %v\n",
				completed, total, result.ObjectKey, result.Error)
		}
	}

	return results
}

// The download worker coroutine
func downloadWorker(client *oss.Client, bucketName string, taskChan <-chan DownloadTask,
	resultChan chan<- DownloadResult, wg *sync.WaitGroup) {
	defer wg.Done()

	for task := range taskChan {
		result := DownloadResult{
			ObjectKey: task.ObjectKey,
			Size:      task.Size,
		}

		// Create the local file directory. 
		if err := os.MkdirAll(filepath.Dir(task.LocalPath), 0755); err != nil {
			result.Error = fmt.Errorf("failed to create directory: %w", err)
			resultChan <- result
			continue
		}

		// Verifies if the file exists and has consistent size.
		if fileInfo, err := os.Stat(task.LocalPath); err == nil {
			if fileInfo.Size() == task.Size {
				result.Success = true
				resultChan <- result
				continue
			}
		}

		// Create the download request.
		getRequest := &oss.GetObjectRequest{
			Bucket: oss.Ptr(bucketName),
			Key:    oss.Ptr(task.ObjectKey),
		}

		// Execute the download operation.
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
		_, err := client.GetObjectToFile(ctx, getRequest, task.LocalPath)
		cancel()

		if err != nil {
			result.Error = err
		} else {
			result.Success = true
		}

		resultChan <- result
	}
}

// Formats the byte count into a human-readable representation.
func formatBytes(bytes int64) string {
	const unit = 1024
	if bytes < unit {
		return fmt.Sprintf("%d B", bytes)
	}
	div, exp := int64(unit), 0
	for n := bytes / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}

# Build the test code into an executable file.
go build -o oss-batch-download main.go

# Downloads the specified folder.
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix images/2024/

# Customize the local directory and concurrency level.
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix documents/ -local-dir ./downloads -workers 10

# Download the whole bucket.
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix ""

Start batch download，bucket: my-bucket, prefix: images/2024/, local directory: ./downloads
Found 150 files to download
✓ [1/150] images/2024/photo1.jpg (2.3 MB)
✓ [2/150] images/2024/photo2.png (1.8 MB)
...
Download complete! success: 148, failures: 2, total size: 1.2 GB