Object Storage Service:Perform MD5 validation during a multipart upload in OSS

Overview

The PutObject, AppendObject, PostObject, MultipartUpload, and UploadPart operations in Object Storage Service (OSS) support MD5 validation during uploads. For the PutObject, AppendObject, and PostObject operations, you can set the Content-MD5 parameter in the request metadata. The parameter value is the MD5 hash of the file calculated by the client. For a multipart upload, MD5 validation is performed on each part. This topic uses the Java software development kit (SDK) as an example to demonstrate how to perform MD5 validation for a multipart upload.

Details

When you use the Java SDK for a multipart upload in OSS, MD5 validation is performed on each part. To do this, call setMd5Digest in UploadPartRequest to set the MD5 hash of the part calculated by the client. The following code provides an example.

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.codec.binary.Hex;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.common.utils.BinaryUtil;
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
import com.aliyun.oss.model.CompleteMultipartUploadResult;
import com.aliyun.oss.model.InitiateMultipartUploadRequest;
import com.aliyun.oss.model.InitiateMultipartUploadResult;
import com.aliyun.oss.model.PartETag;
import com.aliyun.oss.model.UploadPartRequest;
import com.aliyun.oss.model.UploadPartResult;
import com.sun.org.apache.xerces.internal.impl.dv.util.Base64;
import sun.security.provider.MD5;
public class MultipartExample {
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
// The endpoint used in this example is for the China (Hangzhou) region. Replace it with the actual endpoint.
String endpoint = "http://oss-cn-hangzhou.aliyuncs.com";
// An AccessKey pair of an Alibaba Cloud account has full permissions on all API operations. This poses a high security threat. We recommend that you create and use a Resource Access Management (RAM) user to make API calls or perform O&M. To create a RAM user, log on to the RAM console.
String accessKeyId = "your_access_key_id";
String accessKeySecret = "your_access_key_secret";
String bucketName = "your-bucket-name";
// <yourObjectName> specifies the full path of the object to upload to OSS, including the file extension. For example, abc/efg/123.jpg.
String objectName = "your-object-name";
String localFile = "path/to/your-local-file";
// Create an OSSClient instance.
OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
// Create an InitiateMultipartUploadRequest object.
InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName);
// To set the storage class of the object when you initialize the multipart upload, see the following sample code.
// ObjectMetadata metadata = new ObjectMetadata();
// metadata.setHeader(OSSHeaders.OSS_STORAGE_CLASS, StorageClass.Standard.toString());
// request.setObjectMetadata(metadata);
// Initialize the multipart upload.
InitiateMultipartUploadResult upresult = ossClient.initiateMultipartUpload(request);
// The upload ID is returned. It is the unique identifier for the multipart upload event. You can use this upload ID to perform related operations, such as canceling or querying the multipart upload.
String uploadId = upresult.getUploadId();
// partETags is a collection of PartETag objects. A PartETag consists of the ETag and part number of a part.
List<PartETag> partETags = new ArrayList<PartETag>();
// Calculate the number of parts.
final long partSize = 1 * 1024 * 1024L;   // 1 MB
final File sampleFile = new File(localFile);
long fileLength = sampleFile.length();
int partCount = (int) (fileLength / partSize);
if (fileLength % partSize != 0) {
partCount++;
}
// Traverse the parts and upload them.
for (int i = 0; i < partCount; i++) {
    long startPos = i * partSize;
    long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;
    InputStream instream = new FileInputStream(sampleFile);
    InputStream instream1 = new FileInputStream(sampleFile);
// Skip the uploaded parts.
    instream.skip(startPos);
    instream1.skip(startPos);
    String md5;
    if(i==partCount-1){
// Note: For the last part, data is read to the end-of-file, not for the size of a full part.
    md5 = md5(instream1,fileLength - startPos);
    }else{
    md5 = md5(instream1,partSize);
    }
//instream1.skip(n)
    UploadPartRequest uploadPartRequest = new UploadPartRequest();
    uploadPartRequest.setBucketName(bucketName);
    uploadPartRequest.setKey(objectName);
    uploadPartRequest.setUploadId(uploadId);
    uploadPartRequest.setInputStream(instream);
    uploadPartRequest.setMd5Digest(md5);
// Set the part size. The minimum size for a part is 100 KB, except for the last part, which has no size limit.
    uploadPartRequest.setPartSize(curPartSize);
// Set the part number. Each uploaded part has a part number that ranges from 1 to 10,000. If the part number is outside this range, OSS returns the InvalidArgument error code.
    uploadPartRequest.setPartNumber( i + 1);
// Parts do not need to be uploaded in sequence. They can even be uploaded from different clients. OSS assembles the complete object based on the part numbers.
    UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
//System.out.println("server md5" +uploadPartResult.getETag());
// After each part is uploaded, the response from OSS contains a PartETag. The PartETag is saved in partETags.
    partETags.add(uploadPartResult.getPartETag());
}
// Create a CompleteMultipartUploadRequest object.
// When you complete the multipart upload, you must provide all valid PartETags. After OSS receives the submitted PartETags, it verifies each part. After all parts are verified, OSS combines them into a complete object.
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, partETags);
// To set access permissions for the object when you complete the upload, see the following sample code.
// completeMultipartUploadRequest.setObjectACL(CannedAccessControlList.PublicRead);
// Complete the upload.
CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest);
// Shut down the OSSClient.
ossClient.shutdown();
}
public static String md5(InputStream in , long length1) throws Exception{
    byte[] bytes = new byte[(int) length1];

    long length_tmp = length1;

    int readSize = in.read(bytes, (int) 0, (int) length_tmp);

        return BinaryUtil.toBase64String(BinaryUtil.calculateMd5(bytes));
}
}

References

For more information about multipart uploads, see Multipart upload.

Applies to

• Object Storage Service (OSS)