This topic provides code examples for running a migration task from scratch and retrying failed files.
Scenario 1: Run a migration task from scratch
Create a tunnel. Skip this step if you do not need an agent.
Create an agent. Skip this step if you do not need an agent.
Create source and destination data addresses.
Create a migration task.
Run the task and view the results.
Note
If you do not use an agent, skip the part related to tunnel and agent creation in the following code. You must deploy an agent before using the agent. For more information, see Agent management.
package sample;
import com.aliyun.hcs_mgw20240626.Client;
import com.aliyun.hcs_mgw20240626.models.*;
import com.google.gson.Gson;
import java.util.LinkedList;
import java.util.List;
public class Demo {
/** Do not save the AccessKey ID and AccessKey secret in your project code. Otherwise, the AccessKey pair may be leaked, which compromises the security of all resources in your account.*/
static String accessKeyId = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID");
static String accessKeySecret = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET");
/** Enter the ID of your Alibaba Cloud account.*/
static String userId = "11470***876***55";
private static final String AVAILABLE = "available";
public static void main(String[] args) {
try {
/** Step 1: Initialization */
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config();
// This example uses the China (Beijing) region.
config.setEndpoint("cn-beijing.mgw.aliyuncs.com");
config.setAccessKeyId(accessKeyId);
config.setAccessKeySecret(accessKeySecret);
// Both HTTPS and HTTP are supported. If you do not specify a protocol, HTTPS is used by default.
config.setProtocol("http");
Client client = new Client(config);
//* Create a channel and agents. You can associate multiple agents with one channel. */
CreateTunnelRequest createTunnelRequest = new CreateTunnelRequest();
CreateTunnelInfo createTunnelInfo = new CreateTunnelInfo();
TunnelQos qos = new TunnelQos();
// The default values for MaxBandwidth and MaxQps are 0, which indicates no limits. The unit of MaxBandwidth is bit. Set the parameters as needed.
qos.setMaxBandwidth(1073741824L);
qos.setMaxQps(1000);
createTunnelInfo.setTunnelQos(qos);
createTunnelRequest.setImportTunnel(createTunnelInfo);
CreateTunnelResponse createTunnelResponse = client.createTunnel(userId, createTunnelRequest);
String tunnelId = createTunnelResponse.headers.get("x-oss-import-tunnel-id");
// Enter the agent name.
String agentName = "exampleagent";
// Set the network type. Set the value to public for the public network, or vpc for a leased line or VPN.
String agentEndpoint = "public";
// Set the deployment method. Currently, only default is supported.
String deployMethod = "default";
CreateAgentRequest createAgentRequest = new CreateAgentRequest();
CreateAgentInfo createAgentInfo = new CreateAgentInfo();
createAgentInfo.setName(agentName);
createAgentInfo.setTunnelId(tunnelId);
createAgentInfo.setAgentEndpoint(agentEndpoint);
createAgentInfo.setDeployMethod(deployMethod);
createAgentRequest.setImportAgent(createAgentInfo);
client.createAgent(userId, createAgentRequest);
/** Step 2: Create source and destination data addresses and verify their availability. This example shows how to create OSS source and destination addresses. For examples of other sources, see the "Create a data address" section in the Data addresses topic. */
String srcAddress = "examplesrcaddress";
String destAddress = "exampledestaddress";
AddressDetail addrssdetail = new AddressDetail();
addrssdetail.addressType = "oss";
// Set the following parameters based on your actual requirements.
addrssdetail.bucket = "examplebucket";
addrssdetail.prefix = "***/";
addrssdetail.regionId = "oss-cn-beijing";
addrssdetail.role = "rolename_xxxxx";
// If no agent is associated, you can leave this parameter empty.
//addrssdetail.agentList = agentName;
CreateAddressRequest createAddressRequest = new CreateAddressRequest();
CreateAddressInfo createAddressInfo = new CreateAddressInfo();
createAddressInfo.name = srcAddress;
createAddressInfo.setAddressDetail(addrssdetail);
createAddressRequest.setImportAddress(createAddressInfo);
client.createAddress(userId, createAddressRequest);
// Verify the availability of the source data address.
VerifyAddressResponse verifyAddressResponse = client.verifyAddress(userId, srcAddress);
if (!AVAILABLE.equals(verifyAddressResponse.body.verifyAddressResponse.status)) {
System.out.printf("job status is %s", verifyAddressResponse.body.verifyAddressResponse.status);
return;
}
addrssdetail = new AddressDetail();
addrssdetail.addressType = "oss";
// Set the following parameters based on your actual requirements.
addrssdetail.bucket = "examplebucket";
addrssdetail.prefix = "***/";
addrssdetail.regionId = "oss-cn-beijing";
addrssdetail.role = "rolename_xxxxx";
createAddressRequest = new CreateAddressRequest();
createAddressInfo = new CreateAddressInfo();
createAddressInfo.name = destAddress;
createAddressInfo.setAddressDetail(addrssdetail);
createAddressRequest.setImportAddress(createAddressInfo);
client.createAddress(userId, createAddressRequest);
verifyAddressResponse = client.verifyAddress(userId, destAddress);
if (!AVAILABLE.equals(verifyAddressResponse.body.verifyAddressResponse.status)) {
System.out.printf("job status is %s", verifyAddressResponse.body.verifyAddressResponse.status);
return;
}
/** Step 3: Create and start the task */
// Enter the task name.
String jobName = "examplejob";
CreateJobInfo createJobInfo = new CreateJobInfo();
createJobInfo.name = jobName;
createJobInfo.srcAddress = srcAddress;
createJobInfo.destAddress = destAddress;
/*
The overwriteMode and transferMode parameters must be used in combination. The following list describes the valid combinations:
always and all: Overwrites all files.
always and lastmodified: Overwrites files based on their last modification time.
never and "": Does not overwrite files.
*/
/* The file overwrite mode. Valid values: 1. never 2. always */
createJobInfo.overwriteMode = "always";
/* The file transfer mode. Valid values: 1. changed 2. all 3. lastmodified */
createJobInfo.transferMode = "lastmodified";
// If you set maxImportTaskQps to 0 or leave it empty, the default value is used. If you set MaxBandWidth to 0 or leave it empty, the default value is used. The unit of maxBandWidth is bit.
ImportQos importQos = new ImportQos();
// Set maxBandWidth and maxImportTaskQps as needed.
importQos.maxBandWidth = 1073741824L;
importQos.maxImportTaskQps = 1000L;
createJobInfo.setImportQos(importQos);
// Configure the scheduling rule. For more information about the parameters, see the API documentation.
ScheduleRule scheduleRule = new ScheduleRule();
// Set maxScheduleCount, startCronExpression, and suspendCronExpression as needed.
scheduleRule.maxScheduleCount = 2L;
scheduleRule.startCronExpression = "0 0 10 * * ?";
scheduleRule.suspendCronExpression = "0 0 14 * * ?";
createJobInfo.setScheduleRule(scheduleRule);
// Configure the filter rule. For more information about the parameters, see the API documentation.
FilterRule filterRule = new FilterRule();
// The file filter. Set this filter as needed.
KeyFilters keyFilters = new KeyFilters();
KeyFilterItem includekeyFilterItem = new KeyFilterItem();
List<String> includeRegex = new LinkedList<>();
includeRegex.add(".*.jpg");
includeRegex.add(".*.jpeg");
includekeyFilterItem.setRegex(includeRegex);
KeyFilterItem excludekeyFilterItem = new KeyFilterItem();
List<String> excludeRegex = new LinkedList<>();
excludeRegex.add(".*.txt");
excludeRegex.add(".*.js");
excludekeyFilterItem.setRegex(excludeRegex);
keyFilters.setIncludes(includekeyFilterItem);
keyFilters.setExcludes(excludekeyFilterItem);
filterRule.setKeyFilters(keyFilters);
// The time filter. The time must be in UTC format. Set this filter as needed.
LastModifiedFilters lastModifiedFilters = new LastModifiedFilters();
LastModifyFilterItem includeLastModifyFilterItem = new LastModifyFilterItem();
List<TimeFilter> includeRegexs = new LinkedList<>();
TimeFilter includeTimeFilter = new TimeFilter();
includeTimeFilter.startTime = "2006-01-01T00:00:00Z";
includeTimeFilter.endTime = "2006-12-31T23:59:59Z";
includeRegexs.add(includeTimeFilter);
includeLastModifyFilterItem.setTimeFilter(includeRegexs);
LastModifyFilterItem excludeLastModifyFilterItem = new LastModifyFilterItem();
List<TimeFilter> excludeRegexs = new LinkedList<>();
TimeFilter excludeTimeFilter = new TimeFilter();
excludeTimeFilter.startTime = "2008-01-01T00:00:00Z";
excludeTimeFilter.endTime = "2008-12-31T23:59:59Z";
excludeRegexs.add(excludeTimeFilter);
excludeLastModifyFilterItem.setTimeFilter(excludeRegexs);
lastModifiedFilters.setIncludes(includeLastModifyFilterItem);
lastModifiedFilters.setExcludes(excludeLastModifyFilterItem);
filterRule.setLastModifiedFilters(lastModifiedFilters);
createJobInfo.setFilterRule(filterRule);
CreateJobRequest createJobRequest = new CreateJobRequest();
createJobRequest.setImportJob(createJobInfo);
client.createJob(userId, createJobRequest);
UpdateJobRequest updateJobRequest = new UpdateJobRequest();
UpdateJobInfo updateJobInfo = new UpdateJobInfo();
updateJobInfo.setStatus("IMPORT_JOB_LAUNCHING");
updateJobRequest.setImportJob(updateJobInfo);
client.updateJob(userId, jobName, updateJobRequest);
/** Step 4: Poll the current task status */
for(;;) {
GetJobResponse response = client.getJob(userId, jobName, new GetJobRequest());
if ("IMPORT_JOB_INTERRUPTED".equals(response.getBody().importJob.status)) {
System.out.println("job is interrupted");
return;
}
if ("IMPORT_JOB_FINISHED".equals(response.getBody().importJob.status)) {
System.out.println("job is finished");
break;
}
Thread.sleep(6000);
}
/** Step 5: View the results after the task is complete. */
ListJobHistoryRequest listJobHistoryRequest = new ListJobHistoryRequest();
ListJobHistoryResponse listJobHistoryResp = client.listJobHistory(userId, jobName, listJobHistoryRequest);
System.out.println(new Gson().toJson(listJobHistoryResp.body.jobHistoryList.jobHistory))
} catch (Exception e) {
e.printStackTrace();
}
}
}Scenario 2: Retry failed files
After a migration task is complete, some files may fail to migrate. The migration service creates a list of these failed files. The following code shows how to retry these failed files through these steps:
Obtain the details of the failed file list.
Create a new data address.
Create a retry subtask.
Run the subtask and view the results.
Note
Retrying failed files through the API is complex. We recommend that you perform this operation through the console.
package sample;
import com.aliyun.hcs_mgw20240626.Client;
import com.aliyun.hcs_mgw20240626.models.*;
public class Demo {
/** Do not save the AccessKey ID and AccessKey secret in your project code. Otherwise, the AccessKey pair may be leaked, which compromises the security of all resources in your account.*/
static String accessKeyId = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID");
static String accessKeySecret = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET");
/** Enter the ID of your Alibaba Cloud account.*/
static String userId = "11470***876***55";
private static final String AVAILABLE = "available";
private static final String LAUNCHING = "IMPORT_JOB_LAUNCHING";
public static void main(String[] args) {
try {
/** Step 1: Initialization */
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config();
// This example uses the China (Beijing) region.
config.setEndpoint("cn-beijing.mgw.aliyuncs.com");
config.setAccessKeyId(accessKeyId);
config.setAccessKeySecret(accessKeySecret);
// Both HTTPS and HTTP are supported. If you do not specify a protocol, HTTPS is used by default.
config.setProtocol("http");
Client client = new Client(config);
// Set the following parameters as needed.
int runtimeId = 1;
String jobName = "examplejob";
String srcAddress = "examplesrcaddress";
String destAddress = "exampledestaddress";
/** Step 2: Poll the task status to check whether the task is finished or interrupted. */
for(;;) {
GetJobResponse response = client.getJob(userId, jobName, new GetJobRequest());
if ("IMPORT_JOB_INTERRUPTED".equals(response.getBody().importJob.status)) {
System.out.println("job is interrupted");
return;
}
if ("IMPORT_JOB_FINISHED".equals(response.getBody().importJob.status)) {
System.out.println("job is finished");
break;
}
Thread.sleep(6000);
}
/** Step 3: Call ListJobHistory to check for failed files. */
ListJobHistoryRequest listJobHistoryRequest = new ListJobHistoryRequest();
listJobHistoryRequest.runtimeId = runtimeId;
ListJobHistoryResponse listJobHistoryResponse = client.listJobHistory(userId, jobName, listJobHistoryRequest);
if (listJobHistoryResponse.body.jobHistoryList.jobHistory.size() < 1) {
System.out.println("list job history length less than 1");
return;
}
if (listJobHistoryResponse.body.jobHistoryList.jobHistory.get(0).failedCount <= 0) {
System.out.println("job no need retry");
return;
}
/** Step 4: Poll GetJobResult and wait for the Ready state. */
for (;;) {
GetJobResultRequest getJobResultRequest = new GetJobResultRequest();
getJobResultRequest.setRuntimeId(runtimeId);
GetJobResultResponse getJobResultResponse = client.getJobResult(userId, jobName, getJobResultRequest);
if ("NoNeed".equals(getJobResultResponse.body.importJobResult.readyRetry)) {
System.out.println("job no need retry");
return;
}
if ("Ready".equals(getJobResultResponse.body.importJobResult.readyRetry)) {
break;
}
Thread.sleep(6000);
}
retry(client, jobName, srcAddress, destAddress, runtimeId);
} catch (Exception e) {
e.printStackTrace();
}
}
static void retry(Client client, String jobName, String srcAddress, String destAddress, int runtimeId) throws Exception {
/** Step 5: Create a new data address based on the failed file checklist and verify its availability. */
GetJobResultRequest getJobResultRequest = new GetJobResultRequest();
getJobResultRequest.runtimeId = runtimeId;
GetJobResultResponse getJobResultResponse = client.getJobResult(userId, jobName, getJobResultRequest);
GetJobResponse getJobResponse = client.getJob(userId, jobName, new GetJobRequest());
GetAddressResponse getAddressResponse = client.getAddress(userId, srcAddress);
CreateAddressRequest request = new CreateAddressRequest();
CreateAddressInfo info = new CreateAddressInfo();
long time = System.currentTimeMillis();
info.name = srcAddress + "_" + "retry" + "_" + time;
AddressDetail detail = new AddressDetail();
detail.addressType = getJobResultResponse.body.importJobResult.addressType;
detail.invDomain = getJobResultResponse.body.importJobResult.invDomain;
detail.invLocation = getJobResultResponse.body.importJobResult.invLocation;
detail.invBucket = getJobResultResponse.body.importJobResult.invBucket;
detail.invAccessId = getJobResultResponse.body.importJobResult.invAccessId;
detail.invAccessSecret = getJobResultResponse.body.importJobResult.invAccessSecret;
detail.invPath = getJobResultResponse.body.importJobResult.invPath;
detail.invRegionId = getJobResultResponse.body.importJobResult.invRegionId;
detail.domain = getAddressResponse.body.importAddress.addressDetail.domain;
detail.regionId = getAddressResponse.body.importAddress.addressDetail.regionId;
detail.accessId = getAddressResponse.body.importAddress.addressDetail.accessId;
detail.accessSecret = getAddressResponse.body.importAddress.addressDetail.accessSecret;
detail.agentList = getAddressResponse.body.importAddress.addressDetail.agentList;
detail.role = getAddressResponse.body.importAddress.addressDetail.role;
detail.prefix = getAddressResponse.body.importAddress.getAddressDetail().prefix;
detail.bucket = getAddressResponse.body.importAddress.addressDetail.bucket;
info.setAddressDetail(detail);
request.setImportAddress(info);
client.createAddress(userId, request);
VerifyAddressResponse verifyAddressResponse = client.verifyAddress(userId, info.name);
if (!verifyAddressResponse.body.verifyAddressResponse.status.equals(AVAILABLE)) {
throw new Exception("retry srcaddress unavailable");
}
/** Step 6: Create and start a retry subtask. */
CreateJobRequest createJobRequest = new CreateJobRequest();
CreateJobInfo createJobInfo = new CreateJobInfo();
createJobInfo.name = jobName + "_retry_" + time;
createJobInfo.transferMode = getJobResponse.body.importJob.transferMode;
createJobInfo.overwriteMode = getJobResponse.body.importJob.overwriteMode;
createJobInfo.srcAddress = info.name;
createJobInfo.destAddress = destAddress;
createJobInfo.parentVersion = getJobResultResponse.body.importJobResult.version;
createJobInfo.audit = getJobResponse.body.importJob.audit;
createJobInfo.createReport = getJobResponse.body.importJob.createReport;
createJobRequest.setImportJob(createJobInfo);
client.createJob(userId, createJobRequest);
UpdateJobRequest updateJobRequest = new UpdateJobRequest();
UpdateJobInfo updateJobInfo = new UpdateJobInfo();
updateJobInfo.status = LAUNCHING;
updateJobRequest.setImportJob(updateJobInfo);
client.updateJob(userId, createJobInfo.name, updateJobRequest);
}
}