:Download objects as files by using OSS SDK for Go

When you download a single object from a bucket, you can specify conditions based on the last modified time or the ETag of the object. The object is downloaded only if these conditions are met. Otherwise, an error is returned, and the download operation is not triggered. This reduces unnecessary network transmission and resource consumption, and improves download efficiency.

The following table describes the available conditions that you can specify:

The following sample code demonstrates how to implement a conditional download:

package main

import (
	"context"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables
var (
	region     string // The region
	bucketName string // The name of the bucket
	objectName string // The name of the object
)

// Specify the init function used to initialize command line parameters
func init() {
	flag.StringVar(®ion, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&objectName, "object", "", "The name of the object.")
}

func main() {
	// Parse the command line parameters
	flag.Parse()

	// Check whether the bucket name is empty
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	// Check whether the region is empty
	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Check whether the object name is empty
	if len(objectName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, object name required")
	}

	// Load the default configurations and specify the credential provider and region
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSSClient instance
	client := oss.NewClient(cfg)

	// Specify the path of the local file
	localFile := "download.file"

	// For example, an object was last modified at 18:43:02, November 21, 2023. If the time specified in the IfModifiedSince condition is earlier than the last modified time, the object is downloaded.
	date := time.Date(2024, time.October, 21, 18, 43, 2, 0, time.UTC)

	// Assume that the ETag of an object is e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855. If the ETag specified in the IfMatch condition matches the ETag of the object, the object is downloaded.
	etag := "\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\""

	// Create a request to download the object
	getRequest := &oss.GetObjectRequest{
		Bucket:          oss.Ptr(bucketName),                   // The name of the bucket
		Key:             oss.Ptr(objectName),                   // The name of the object
		IfModifiedSince: oss.Ptr(date.Format(http.TimeFormat)), // Specify the IfModifiedSince condition
		IfMatch:         oss.Ptr(etag),                         // Specify the IfMatch condition
	}

	// Download the object to your local computer and display the download result
	result, err := client.GetObjectToFile(context.TODO(), getRequest, localFile)
	if err != nil {
		log.Fatalf("failed to get object to file %v", err)
	}

	log.Printf("get object to file result:%#v\n", result)
}

When you download an object, you can use the progress bar to view the download progress in real time, which helps you check whether the download task is stuck after waiting for the object download for a long time.

The following sample code demonstrates how to monitor the download progress by displaying a progress bar.

package main

import (
	"context"
	"flag"
	"fmt"
	"log"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables
var (
	region     string // The region
	bucketName string // The name of the bucket
	objectName string // The name of the object
)

// Specify the init function used to initialize command line parameters
func init() {
	flag.StringVar(®ion, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&objectName, "object", "", "The name of the object.")
}

func main() {
	// Parse the command line arguments
	flag.Parse()

	// Check whether the bucket name is empty
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	// Check whether the region is empty
	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Check whether the object name is empty
	if len(objectName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, object name required")
	}

	// Load the default configurations and specify the credential provider and region
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSSClient instance
	client := oss.NewClient(cfg)

	// Specify the path of the local file
	localFile := "download.file"

	// Create a request to download the object
	getRequest := &oss.GetObjectRequest{
		Bucket: oss.Ptr(bucketName), // The name of the bucket
		Key:    oss.Ptr(objectName), // The name of the object
		ProgressFn: func(increment, transferred, total int64) {
			fmt.Printf("increment:%v, transferred:%v, total:%v\n", increment, transferred, total)
		}, // Display the download progress
	}

	// Download the object to a local device and process the result
	result, err := client.GetObjectToFile(context.TODO(), getRequest, localFile)
	if err != nil {
		log.Fatalf("failed to get object to file %v", err)
	}

	log.Printf("get object to file result:%#v\n", result)
}

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
	"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)

// Define the global variables
var (
	region     string // The region
	bucketName string // The name of the bucket
	prefix     string // The object prefix (folder path)
	localDir   string // The local download directory
	maxWorkers int    // The maximum concurrency
	maxKeys    int    // The maximum number of objects to list at once
)

// DownloadTask download task structure
type DownloadTask struct {
	ObjectKey string
	LocalPath string
	Size      int64
}

// DownloadResult download result structure
type DownloadResult struct {
	ObjectKey string
	Success   bool
	Error     error
	Size      int64
}

// Use the init function to initialize parameters
func init() {
	flag.StringVar(®ion, "region", "", "The region in which the bucket is located.")
	flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
	flag.StringVar(&prefix, "prefix", "", "The prefix (folder path) to download.")
	flag.StringVar(&localDir, "local-dir", "./downloads", "Local directory to save downloaded files.")
	flag.IntVar(&maxWorkers, "workers", 5, "Maximum number of concurrent downloads.")
	flag.IntVar(&maxKeys, "max-keys", 1000, "Maximum number of objects to list at once.")
}

func main() {
	// Parse parameters
	flag.Parse()

	// Check required parameters
	if len(bucketName) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, bucket name required")
	}

	if len(region) == 0 {
		flag.PrintDefaults()
		log.Fatalf("invalid parameters, region required")
	}

	// Ensure the prefix ends with / (if not empty string)
	if prefix != "" && !strings.HasSuffix(prefix, "/") {
		prefix += "/"
	}

	// Create the local download directory
	if err := os.MkdirAll(localDir, 0755); err != nil {
		log.Fatalf("failed to create local directory: %v", err)
	}

	// Load the default configurations and specify the credential provider and region
	cfg := oss.LoadDefaultConfig().
		WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
		WithRegion(region)

	// Create an OSS client
	client := oss.NewClient(cfg)

	fmt.Printf("Starting batch download, bucket: %s, prefix: %s, local directory: %s\n", bucketName, prefix, localDir)

	// List all objects to download
	tasks, err := listObjects(client, bucketName, prefix)
	if err != nil {
		log.Fatalf("failed to list objects: %v", err)
	}

	if len(tasks) == 0 {
		fmt.Println("No files found to download")
		return
	}

	fmt.Printf("Found %d files to download\n", len(tasks))

	// Execute batch download
	results := batchDownload(client, tasks, maxWorkers)

	// Summarize download results
	var successCount, failCount int
	var totalSize int64
	for _, result := range results {
		if result.Success {
			successCount++
			totalSize += result.Size
		} else {
			failCount++
			fmt.Printf("Download failed: %s, error: %v\n", result.ObjectKey, result.Error)
		}
	}

	fmt.Printf("\nDownload completed! Success: %d, Failed: %d, Total size: %s\n",
		successCount, failCount, formatBytes(totalSize))
}

// listObjects lists all objects with the specified prefix in the bucket
func listObjects(client *oss.Client, bucketName, prefix string) ([]DownloadTask, error) {
	var tasks []DownloadTask
	var continuationToken *string

	for {
		// Create a request to list objects
		request := &oss.ListObjectsV2Request{
			Bucket:            oss.Ptr(bucketName),
			Prefix:            oss.Ptr(prefix),
			MaxKeys:           int32(maxKeys),
			ContinuationToken: continuationToken,
		}

		// Execute the list operation
		result, err := client.ListObjectsV2(context.TODO(), request)
		if err != nil {
			return nil, fmt.Errorf("failed to list objects: %w", err)
		}

		// Process the list results
		for _, obj := range result.Contents {
			// Skip folder objects (ending with / and size is 0)
			if strings.HasSuffix(*obj.Key, "/") && obj.Size == 0 {
				continue
			}

			// Calculate the local file path
			relativePath := strings.TrimPrefix(*obj.Key, prefix)
			localPath := filepath.Join(localDir, relativePath)

			tasks = append(tasks, DownloadTask{
				ObjectKey: *obj.Key,
				LocalPath: localPath,
				Size:      obj.Size,
			})
		}

		// Check if there are more objects
		if result.NextContinuationToken == nil {
			break
		}
		continuationToken = result.NextContinuationToken
	}

	return tasks, nil
}

// batchDownload executes batch download
func batchDownload(client *oss.Client, tasks []DownloadTask, maxWorkers int) []DownloadResult {
	taskChan := make(chan DownloadTask, len(tasks))
	resultChan := make(chan DownloadResult, len(tasks))

	// Start worker goroutines
	var wg sync.WaitGroup
	for i := 0; i < maxWorkers; i++ {
		wg.Add(1)
		go downloadWorker(client, bucketName, taskChan, resultChan, &wg)
	}

	// Send download tasks
	go func() {
		for _, task := range tasks {
			taskChan <- task
		}
		close(taskChan)
	}()

	// Wait for all worker goroutines to complete
	go func() {
		wg.Wait()
		close(resultChan)
	}()

	// Collect results and display progress
	var results []DownloadResult
	completed := 0
	total := len(tasks)

	for result := range resultChan {
		results = append(results, result)
		completed++

		if result.Success {
			fmt.Printf("✓ [%d/%d] %s (%s)\n",
				completed, total, result.ObjectKey, formatBytes(result.Size))
		} else {
			fmt.Printf("✗ [%d/%d] %s - Error: %v\n",
				completed, total, result.ObjectKey, result.Error)
		}
	}

	return results
}

// downloadWorker download worker goroutine
func downloadWorker(client *oss.Client, bucketName string, taskChan <-chan DownloadTask,
	resultChan chan<- DownloadResult, wg *sync.WaitGroup) {
	defer wg.Done()

	for task := range taskChan {
		result := DownloadResult{
			ObjectKey: task.ObjectKey,
			Size:      task.Size,
		}

		// Create local file directory
		if err := os.MkdirAll(filepath.Dir(task.LocalPath), 0755); err != nil {
			result.Error = fmt.Errorf("failed to create directory: %w", err)
			resultChan <- result
			continue
		}

		// Check if file already exists and size matches
		if fileInfo, err := os.Stat(task.LocalPath); err == nil {
			if fileInfo.Size() == task.Size {
				result.Success = true
				resultChan <- result
				continue
			}
		}

		// Create download request
		getRequest := &oss.GetObjectRequest{
			Bucket: oss.Ptr(bucketName),
			Key:    oss.Ptr(task.ObjectKey),
		}

		// Execute download
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
		_, err := client.GetObjectToFile(ctx, getRequest, task.LocalPath)
		cancel()

		if err != nil {
			result.Error = err
		} else {
			result.Success = true
		}

		resultChan <- result
	}
}

// formatBytes formats bytes to readable format
func formatBytes(bytes int64) string {
	const unit = 1024
	if bytes < unit {
		return fmt.Sprintf("%d B", bytes)
	}
	div, exp := int64(unit), 0
	for n := bytes / unit; n >= unit; n /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}

Usage

# Compile the test code into an executable file
go build -o oss-batch-download main.go

# Download a specific folder
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix images/2024/

# Customize local directory and concurrency
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix documents/ -local-dir ./downloads -workers 10

# Download the entire bucket
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix ""

Output example

The program displays detailed download progress during execution:

Starting batch download, bucket: my-bucket, prefix: images/2024/, local directory: ./downloads
Found 150 files to download
✓ [1/150] images/2024/photo1.jpg (2.3 MB)
✓ [2/150] images/2024/photo2.png (1.8 MB)
...
Download completed! Success: 148, Failed: 2, Total size: 1.2 GB