Object Storage Service:Download objects to a local device using OSS SDK for Go 2.0

When you download a single object from a bucket, you can specify conditions based on the last modified time or the ETag of the object. The object is downloaded only if these conditions are met. Otherwise, an error is returned, and the download operation is not triggered. This reduces unnecessary network transmission and resource consumption, and improves the download efficiency.

The following table describes the available conditions.

The following sample code provides an example on how to implement a conditional download:

package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables.
var (
	region     string // The region in which the bucket is located.
	bucketName string // The name of the bucket.
	objectName string // The name of the object.
)

// Use the init function to initialize parameters.
func init() {
	flag.StringVar(&region, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&objectName, "object", "", "The name of the object.")
}

func main() {
	// Parse parameters.
	flag.Parse()

	// Check whether the bucket name is empty.
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	// Check whether the region is empty.
	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Check whether the object name is empty.
	if len(objectName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, object name required")
	}

	// Load the default configurations and specify the credential provider and region.
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client.
	client := oss.NewClient(cfg)

	// Specify the path of the local file.
	localFile := "download.file"

	// For example, an object was last modified at 18:43:02, November 21, 2023. If the time specified in the IfModifiedSince condition is earlier than the last modified time, the object is downloaded. 
	date := time.Date(2024, time.October, 21, 18, 43, 2, 0, time.UTC)

	// Assume that the ETag of an object is e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855. If the ETag specified in the IfMatch condition matches the ETag of the object, the object is downloaded. 
	etag := "\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\""

	// Create a request to download the object.
	getRequest := &oss.GetObjectRequest{
		Bucket:          oss.Ptr(bucketName),                   // The name of the bucket.
		Key:             oss.Ptr(objectName),                   // The name of the object.
		IfModifiedSince: oss.Ptr(date.Format(http.TimeFormat)), // Specify the IfModifiedSince condition.
		IfMatch:         oss.Ptr(etag),                         // Specify the IfMatch condition.
	}

	// Download the object to a local device and process the result.
	result, err := client.GetObjectToFile(context.TODO(), getRequest, localFile)
	if err != nil {
		log.Fatalf("failed to get object to file %v", err)
	}

	log.Printf("get object to file result:%#v\n", result)
}

When you download an object, you can use a progress bar to monitor the download progress in real time. Progress monitoring helps you check whether a download task is stuck if it takes a long time to complete.

The following sample code provides an example on how to monitor the download progress by displaying a progress bar.

package main

import (
	"context"
	"flag"
	"fmt"
	"log"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables.
var (
	region     string // The region in which the bucket is located.
	bucketName string // The name of the bucket.
	objectName string // The name of the object.
)

// Use the init function to initialize parameters.
func init() {
	flag.StringVar(&region, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&objectName, "object", "", "The name of the object.")
}

func main() {
	// Parse parameters.
	flag.Parse()

	// Check whether the bucket name is empty.
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	// Check whether the region is empty.
	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Check whether the object name is empty.
	if len(objectName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, object name required")
	}

	// Load the default configurations and specify the credential provider and region.
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client.
	client := oss.NewClient(cfg)

	// Specify the path of the local file.
	localFile := "download.file"

	// Create a request to download the object.
	getRequest := &oss.GetObjectRequest{
		Bucket: oss.Ptr(bucketName), // The name of the bucket.
		Key:    oss.Ptr(objectName), // The name of the object.
		ProgressFn: func(increment, transferred, total int64) {
			fmt.Printf("increment:%v, transferred:%v, total:%v\n", increment, transferred, total)
		}, // Display the download progress.
	}

	// Download the object to a local device and process the result.
	result, err := client.GetObjectToFile(context.TODO(), getRequest, localFile)
	if err != nil {
		log.Fatalf("failed to get object to file %v", err)
	}

	log.Printf("get object to file result:%#v\n", result)
}

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables.
var (
	region     string // The region in which the bucket is located.
	bucketName string // The name of the bucket.
	prefix     string // The prefix (file path) of the object.
	localDir   string // The local directory.
	maxWorkers int    // The maximum concurrency.
	maxKeys    int    // The maximum number of objects that can be listed.
)

// The structure of the download task.
type DownloadTask struct {
	ObjectKey string
	LocalPath string
	Size      int64
}

// The structure of the download result.
type DownloadResult struct {
	ObjectKey string
	Success   bool
	Error     error
	Size      int64
}

// Define the init function used to initialize parameters.
func init() {
	flag.StringVar(&region, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&prefix, "prefix", "", "The prefix (folder path) to download.")
	flag.StringVar(&localDir, "local-dir", "./downloads", "Local directory to save downloaded files.")
	flag.IntVar(&maxWorkers, "workers", 5, "Maximum number of concurrent downloads.")
	flag.IntVar(&maxKeys, "max-keys", 1000, "Maximum number of objects to list at once.")
}

func main() {
	// Parse the command-line parameters.
	flag.Parse()

	// Check the required parameters.
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// To ensure a prefix ends with "/" if not empty.
	if prefix != "" && !strings.HasSuffix(prefix, "/") {
		prefix += "/"
	}

	// Create the local download directory.
	if err := os.MkdirAll(localDir, 0755); err != nil {
		log.Fatalf("failed to create local directory: %v", err)
	}

	// Load the default configurations and specify the credential provider and region.
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client.
	client := oss.NewClient(cfg)

	fmt.Printf("start batch download，bucket: %s, prefix: %s, local directory: %s\n", bucketName, prefix, localDir)

	// List all objects requiring download.
	tasks, err := listObjects(client, bucketName, prefix)
	if err != nil {
		log.Fatalf("failed to list objects: %v", err)
	}

	if len(tasks) == 0 {
		fmt.Println("No objects requiring download were found")
		return
	}

	fmt.Printf("%d objects need to be downloaded\n", len(tasks))

	// Execute batch download.
	results := batchDownload(client, tasks, maxWorkers)

	// Summarize the result of the download.
	var successCount, failCount int
	var totalSize int64
	for _, result := range results {
		if result.Success {
			successCount++
			totalSize += result.Size
		} else {
			failCount++
			fmt.Printf("Download failed: %s, Error: %v\n", result.ObjectKey, result.Error)
		}
	}

	fmt.Printf("\nDownload complete! success: %d, Failure: %d, Total size: %s\n",
		successCount, failCount, formatBytes(totalSize))
}

// List all objects in a bucket with a specified prefix.
func listObjects(client *oss.Client, bucketName, prefix string) ([]DownloadTask, error) {
	var tasks []DownloadTask
	var continuationToken *string

	for {
		// Create a request to list objects.
		request := &oss.ListObjectsV2Request{
			Bucket:            oss.Ptr(bucketName),
			Prefix:            oss.Ptr(prefix),
			MaxKeys:           int32(maxKeys),
			ContinuationToken: continuationToken,
		}

		// Execute the list operation.
		result, err := client.ListObjectsV2(context.TODO(), request)
		if err != nil {
			return nil, fmt.Errorf("failed to list objects: %w", err)
		}

		// Process the result of the list operation.
		for _, obj := range result.Contents {
			// Excluding folder objects ending with '/' and having zero size
			if strings.HasSuffix(*obj.Key, "/") && obj.Size == 0 {
				continue
			}

			// Computes the local file directory.
			relativePath := strings.TrimPrefix(*obj.Key, prefix)
			localPath := filepath.Join(localDir, relativePath)

			tasks = append(tasks, DownloadTask{
				ObjectKey: *obj.Key,
				LocalPath: localPath,
				Size:      obj.Size,
			})
		}

		// Check whether there are more objects.
		if result.NextContinuationToken == nil {
			break
		}
		continuationToken = result.NextContinuationToken
	}

	return tasks, nil
}

// Execute batch download operation.
func batchDownload(client *oss.Client, tasks []DownloadTask, maxWorkers int) []DownloadResult {
	taskChan := make(chan DownloadTask, len(tasks))
	resultChan := make(chan DownloadResult, len(tasks))

	// Launches the worker coroutine.
	var wg sync.WaitGroup
	for i := 0; i < maxWorkers; i++ {
		wg.Add(1)
		go downloadWorker(client, bucketName, taskChan, resultChan, &wg)
	}

	// Dispatch the download task.
	go func() {
		for _, task := range tasks {
			taskChan <- task
		}
		close(taskChan)
	}()

	// Awaits the completion of all worker coroutines.
	go func() {
		wg.Wait()
		close(resultChan)
	}()

	// Collect the results and display the progres.
	var results []DownloadResult
	completed := 0
	total := len(tasks)

	for result := range resultChan {
		results = append(results, result)
		completed++

		if result.Success {
			fmt.Printf("✓ [%d/%d] %s (%s)\n",
				completed, total, result.ObjectKey, formatBytes(result.Size))
		} else {
			fmt.Printf("✗ [%d/%d] %s - Error: %v\n",
				completed, total, result.ObjectKey, result.Error)
		}
	}

	return results
}

// The download worker coroutine
func downloadWorker(client *oss.Client, bucketName string, taskChan <-chan DownloadTask,
	resultChan chan<- DownloadResult, wg *sync.WaitGroup) {
	defer wg.Done()

	for task := range taskChan {
		result := DownloadResult{
			ObjectKey: task.ObjectKey,
			Size:      task.Size,
		}

		// Create the local file directory. 
		if err := os.MkdirAll(filepath.Dir(task.LocalPath), 0755); err != nil {
			result.Error = fmt.Errorf("failed to create directory: %w", err)
			resultChan <- result
			continue
		}

		// Verifies if the file exists and has consistent size.
		if fileInfo, err := os.Stat(task.LocalPath); err == nil {
			if fileInfo.Size() == task.Size {
				result.Success = true
				resultChan <- result
				continue
			}
		}

		// Create the download request.
		getRequest := &oss.GetObjectRequest{
			Bucket: oss.Ptr(bucketName),
			Key:    oss.Ptr(task.ObjectKey),
		}

		// Execute the download operation.
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
		_, err := client.GetObjectToFile(ctx, getRequest, task.LocalPath)
		cancel()

		if err != nil {
			result.Error = err
		} else {
			result.Success = true
		}

		resultChan <- result
	}
}

// Formats the byte count into a human-readable representation.
func formatBytes(bytes int64) string {
	const unit = 1024
	if bytes < unit {
		return fmt.Sprintf("%d B", bytes)
	}
	div, exp := int64(unit), 0
	for n := bytes / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}

Method

# Build the test code into an executable file.
go build -o oss-batch-download main.go

# Downloads the specified folder.
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix images/2024/

# Customize the local directory and concurrency level.
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix documents/ -local-dir ./downloads -workers 10

# Download the whole bucket.
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix ""

Sample output

Displays detailed download progress during execution:

Start batch download，bucket: my-bucket, prefix: images/2024/, local directory: ./downloads
Found 150 files to download
✓ [1/150] images/2024/photo1.jpg (2.3 MB)
✓ [2/150] images/2024/photo2.png (1.8 MB)
...
Download complete! success: 148, failures: 2, total size: 1.2 GB