Object Storage Service:Multipart upload (C SDK)

Notes

• In this topic, the public endpoint of the China (Hangzhou) region is used. If you want to access OSS from other Alibaba Cloud services in the same region as OSS, use an internal endpoint. For more information about OSS regions and endpoints, see Regions and endpoints.

• This topic demonstrates creating an OSSClient instance with an OSS endpoint. For alternative configurations, such as using a custom domain or authenticating with credentials from Security Token Service (STS), see Initialization (C SDK).

• Multipart upload uses InitiateMultipartUpload, UploadPart, and CompleteMultipartUpload. You must have the oss:PutObject permission. Grant custom access policies to a RAM user.

Multipart upload process

To upload an object using multipart upload, perform the following steps:

1. Initialize a multipart upload event.

   Call the oss_init_multipart_upload method. The method returns a globally unique upload ID created by OSS.

2. Upload parts.

   Call oss_upload_part_from_file to upload parts.

   Note

   • The part number identifies the position of a part within the object. Uploading data with the same part number overwrites the existing part.

   • OSS returns the MD5 hash of received part data in the response ETag header.

   • OSS calculates the MD5 hash of uploaded data and compares it with the hash calculated by the SDK. A mismatch returns the InvalidDigest error code.

3. Complete the multipart upload.

   After you upload all the parts, call the oss_complete_multipart_upload method to assemble them into a complete file.

Complete multipart upload example

The following example shows the complete multipart upload process:

#include "oss_api.h"
#include "aos_http_io.h"
#include <sys/stat.h>
/* Set yourEndpoint to the Endpoint of the region where the bucket is located. For example, if the bucket is in the China (Hangzhou) region, set the Endpoint to https://oss-cn-hangzhou.aliyuncs.com. */
const char *endpoint = "yourEndpoint";

/* Enter the bucket name, for example, examplebucket. */
const char *bucket_name = "examplebucket";
/* Enter the full path of the object. The path cannot contain the bucket name. For example, enter exampledir/exampleobject.txt. */
const char *object_name = "exampledir/exampleobject.txt";
/* Enter the full path of the local file. */
const char *local_filename = "yourLocalFilename";
/* Set yourRegion to the region where the bucket is located. For example, if the bucket is in the China (Hangzhou) region, set the Region to cn-hangzhou. */
const char *region = "yourRegion";
void init_options(oss_request_options_t *options)
{
    options->config = oss_config_create(options->pool);
    /* Initialize the aos_string_t type with a char* string. */
    aos_str_set(&options->config->endpoint, endpoint);
    /* Get access credentials from environment variables. Before running this code, make sure that the OSS_ACCESS_KEY_ID and OSS_ACCESS_KEY_SECRET environment variables are set. */    
    aos_str_set(&options->config->access_key_id, getenv("OSS_ACCESS_KEY_ID"));
    aos_str_set(&options->config->access_key_secret, getenv("OSS_ACCESS_KEY_SECRET"));
    // Configure the following two parameters.
    aos_str_set(&options->config->region, region);
    options->config->signature_version = 4;
    /* Specify whether to use a CNAME to access OSS. A value of 0 means a CNAME is not used. */
    options->config->is_cname = 0;
    /* Set network parameters, such as the timeout period. */
    options->ctl = aos_http_controller_create(options->pool, 0);
}
int64_t get_file_size(const char *file_path)
{
    int64_t filesize = -1;
    struct stat statbuff;
    if(stat(file_path, &statbuff) < 0){
        return filesize;
    } else {
        filesize = statbuff.st_size;
    }
    return filesize;
}
int main(int argc, char *argv[])
{
    /* At the program entry point, call the aos_http_io_initialize method to initialize global resources, such as the network and memory. */
    if (aos_http_io_initialize(NULL, 0) != AOSE_OK) {
        exit(1);
    }
    /* A memory pool (pool) for memory management. It is equivalent to apr_pool_t. The implementation code is in the apr library. */
    aos_pool_t *pool;
    /* Create a new memory pool. The second parameter is NULL, which indicates that the pool does not inherit from other memory pools. */
    aos_pool_create(&pool, NULL);
    /* Create and initialize options. This parameter includes global configurations such as endpoint, access_key_id, access_key_secret, is_cname, and curl. */
    oss_request_options_t *oss_client_options;
    /* Allocate memory for options in the memory pool. */
    oss_client_options = oss_request_options_create(pool);
    /* Initialize the client options, oss_client_options. */
    init_options(oss_client_options);
    /* Initialize parameters. */
    aos_string_t bucket;
    aos_string_t object;
    oss_upload_file_t *upload_file = NULL;
    aos_string_t upload_id;   
    aos_table_t *headers = NULL;
    aos_table_t *complete_headers = NULL;
    aos_table_t *resp_headers = NULL;
    aos_status_t *resp_status = NULL; 
    aos_str_set(&bucket, bucket_name);
    aos_str_set(&object, object_name);
    aos_str_null(&upload_id);
    headers = aos_table_make(pool, 1);
    complete_headers = aos_table_make(pool, 1);
    int part_num = 1;
    /* Initialize the multipart upload and obtain an upload ID. */
    resp_status = oss_init_multipart_upload(oss_client_options, &bucket, &object, &upload_id, headers, &resp_headers);
    /* Check whether the multipart upload is successfully initialized. */
    if (aos_status_is_ok(resp_status)) {
        printf("Init multipart upload succeeded, upload_id:%.*s\n", 
               upload_id.len, upload_id.data);
    } else {
        printf("Init multipart upload failed, upload_id:%.*s\n", 
               upload_id.len, upload_id.data);
    }
    /* Upload parts. */
    int64_t file_length = 0;
    int64_t pos = 0;
    aos_list_t complete_part_list;
       oss_complete_part_content_t* complete_content = NULL;
    char* part_num_str = NULL;
    char* etag = NULL;
    aos_list_init(&complete_part_list);
    file_length = get_file_size(local_filename);
    while(pos < file_length) {
        upload_file = oss_create_upload_file(pool);
        aos_str_set(&upload_file->filename, local_filename);
        upload_file->file_pos = pos;
        pos += 100 * 1024;
        upload_file->file_last = pos < file_length ? pos : file_length;
        resp_status = oss_upload_part_from_file(oss_client_options, &bucket, &object, &upload_id, part_num++, upload_file, &resp_headers);

        /* Save the part number and ETag. */
        complete_content = oss_create_complete_part_content(pool);
        part_num_str = apr_psprintf(pool, "%d", part_num-1);
        aos_str_set(&complete_content->part_number, part_num_str);
        etag = apr_pstrdup(pool,
        (char*)apr_table_get(resp_headers, "ETag"));
        aos_str_set(&complete_content->etag, etag);
        aos_list_add_tail(&complete_content->node, &complete_part_list);

        if (aos_status_is_ok(resp_status)) {
            printf("Multipart upload part from file succeeded\n");
        } else {
            printf("Multipart upload part from file failed\n");
        }
    }

    /* Complete the multipart upload. */
    resp_status = oss_complete_multipart_upload(oss_client_options, &bucket, &object, &upload_id,
            &complete_part_list, complete_headers, &resp_headers);
    /* Check whether the multipart upload is complete. */
    if (aos_status_is_ok(resp_status)) {
        printf("Complete multipart upload from file succeeded, upload_id:%.*s\n", 
               upload_id.len, upload_id.data);
    } else {
        printf("Complete multipart upload from file failed\n");
    }
    /* Release the memory pool. This releases the memory allocated for each resource during the request. */
    aos_pool_destroy(pool);
    /* Release the previously allocated global resources. */
    aos_http_io_deinitialize();
    return 0;
}

When you call the oss_complete_multipart_upload operation, you must provide the ETag value of each part. You can obtain these values in one of the following two ways:

• When you upload each part, the response contains the ETag value for that part. You can save this value and use it later. The preceding example uses this method.

• Call the oss_list_upload_part operation to retrieve the ETag values of previously uploaded parts.

Abort a multipart upload event

The following code shows how to abort a multipart upload event.

#include "oss_api.h"
#include "aos_http_io.h"
/* Set yourEndpoint to the endpoint of the region where the bucket is located. For example, if the bucket is in the China (Hangzhou) region, set the endpoint to https://oss-cn-hangzhou.aliyuncs.com. */
const char *endpoint = "yourEndpoint";

/* Specify the bucket name. For example, examplebucket. */
const char *bucket_name = "examplebucket";
/* Specify the full path of the object. The full path cannot contain the bucket name. For example, exampledir/exampleobject.txt. */
const char *object_name = "exampledir/exampleobject.txt";
/* Set yourRegion to the region where the bucket is located. For example, if the bucket is in the China (Hangzhou) region, set the region to cn-hangzhou. */
const char *region = "yourRegion";
void init_options(oss_request_options_t *options)
{
    options->config = oss_config_create(options->pool);
    /* Initialize the aos_string_t type with a char* string. */
    aos_str_set(&options->config->endpoint, endpoint);
    /* Obtain access credentials from environment variables. Before you run this code, make sure that the OSS_ACCESS_KEY_ID and OSS_ACCESS_KEY_SECRET environment variables are set. */    
    aos_str_set(&options->config->access_key_id, getenv("OSS_ACCESS_KEY_ID"));
    aos_str_set(&options->config->access_key_secret, getenv("OSS_ACCESS_KEY_SECRET"));
    // Configure the following two parameters.
    aos_str_set(&options->config->region, region);
    options->config->signature_version = 4;
    /* Specifies whether to use a CNAME to access OSS. 0 indicates that a CNAME is not used. */
    options->config->is_cname = 0;
    /* Set network parameters, such as the timeout period. */
    options->ctl = aos_http_controller_create(options->pool, 0);
}
int main(int argc, char *argv[])
{
    /* Call the aos_http_io_initialize method at the program entry to initialize global resources, such as the network and memory. */
    if (aos_http_io_initialize(NULL, 0) != AOSE_OK) {
        exit(1);
    }
    /* The memory pool (pool) for memory management is equivalent to apr_pool_t. Its implementation code is in the apr library. */
    aos_pool_t *pool;
    /* Create a new memory pool. The second parameter is NULL, which indicates that the new memory pool does not inherit from another memory pool. */
    aos_pool_create(&pool, NULL);
    /* Create and initialize options. This parameter contains global configuration information, such as endpoint, access_key_id, access_key_secret, is_cname, and curl. */
    oss_request_options_t *oss_client_options;
    /* Allocate memory to options in the memory pool. */
    oss_client_options = oss_request_options_create(pool);
    /* Initialize the client options oss_client_options. */
    init_options(oss_client_options);
    /* Initialize parameters. */
    aos_string_t bucket;
    aos_string_t object;
    aos_string_t upload_id;   
    aos_table_t *headers = NULL;
    aos_table_t *resp_headers = NULL;
    aos_status_t *resp_status = NULL; 
    aos_str_set(&bucket, bucket_name);
    aos_str_set(&object, object_name);
    aos_str_null(&upload_id);
    /* Initialize the multipart upload to obtain an upload ID (upload_id). */
    resp_status = oss_init_multipart_upload(oss_client_options, &bucket, &object, &upload_id, headers, &resp_headers);
    /* Check whether the multipart upload is successfully initialized. */
    if (aos_status_is_ok(resp_status)) {
        printf("Init multipart upload succeeded, upload_id:%.*s\n", 
               upload_id.len, upload_id.data);
    } else {
        printf("Init multipart upload failed, upload_id:%.*s\n", 
               upload_id.len, upload_id.data);
    }
    /* Abort the multipart upload. */
    resp_status = oss_abort_multipart_upload(oss_client_options, &bucket, &object, &upload_id, &resp_headers);
    /* Check whether the multipart upload is successfully aborted. */
    if (aos_status_is_ok(resp_status)) {
        printf("Abort multipart upload succeeded, upload_id::%.*s\n", 
               upload_id.len, upload_id.data);
    } else {
        printf("Abort multipart upload failed\n"); 
    }
    /* Release the memory pool. This releases the memory allocated to resources during the request. */
    aos_pool_destroy(pool);
    /* Release the previously allocated global resources. */
    aos_http_io_deinitialize();
    return 0;
}

References

• For a complete code sample of a multipart upload, see the GitHub sample.

• A multipart upload involves three API operations:

  • Initialize a multipart upload: InitiateMultipartUpload.

  • Upload a part: UploadPart.

  • Complete a multipart upload: CompleteMultipartUpload.

• For more information about aborting a multipart upload event, see AbortMultipartUpload.