Export DMS operation logs to Simple Log Service in the console or with code - Data Management

Data Management (DMS) can export operation logs to Simple Log Service (SLS) for centralized storage, querying, data transformation, and analysis.

Prerequisites

Before you begin, make sure you have:

An activated Simple Log Service account. For more information, see Activate Simple Log Service.
An SLS project and a Logstore. For more information, see Manage projects and Create a basic Logstore.
The database instance added to DMS on the Instance Management page. For more information, see Add an ApsaraDB instance.
A Logstore with no existing log entries and either a full-text index or a field index enabled. For more information, see Create an index.
(For Java implementation only) The GetOpLog permission for DMS, and the PutLogs and CreateIndex permissions for SLS.

Billing

Exporting DMS operation logs to SLS is free of charge.
After logs are collected in SLS, charges apply based on the Logstore billing method: if the Logstore uses the Pay-by-feature billing method, you are charged for storage space, read traffic, requests, data transformation, and data shipping. For more information, see the billing documentation.
If the Logstore uses the Pay-by-ingested-data billing method, you are charged for the amount of raw data ingested. For more information, see the billing documentation.

Export logs using Java

As an alternative to the console, export logs programmatically using the Java SDK. The code calls the DMS GetOpLog API to retrieve logs, then writes them to SLS using PutLogs.

Note

(For Java implementation only) The GetOpLog permission for DMS, and the PutLogs and CreateIndex permissions for SLS.
Avoid hardcoding an AccessKey in your project code. For more information about configuring credentials securely, see Manage access credentials.

Add the following Maven dependencies:

<dependency>
  <groupId>com.aliyun</groupId>
  <artifactId>dms_enterprise20181101</artifactId>
  <version>2.0.0</version>
</dependency>

<dependency>
  <groupId>com.aliyun.openservices</groupId>
  <artifactId>aliyun-log</artifactId>
  <version>0.6.100</version>
</dependency>

package org.example;


import com.aliyun.dms_enterprise20181101.models.GetOpLogResponse;
import com.aliyun.dms_enterprise20181101.models.GetOpLogResponseBody;
import com.aliyun.openservices.log.common.LogItem;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Optional;


public class ExportDmsOperLogExample {

    private static com.aliyun.dms_enterprise20181101.Client dmsClient = null;

    private static com.aliyun.openservices.log.Client slsClient = null;

    public static com.aliyun.credentials.Client getCredentialClient() {
        com.aliyun.credentials.models.Config credentialConfig = new com.aliyun.credentials.models.Config();
        credentialConfig.setType("access_key");
        // Obtain the AccessKey ID from an environment variable.
        credentialConfig.setAccessKeyId(System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID"));
        // Obtain the AccessKey secret from an environment variable.
        credentialConfig.setAccessKeySecret(System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET"));

        return new com.aliyun.credentials.Client(credentialConfig);
    }

    // Create a DMS client to call GetOpLog.
    public static synchronized com.aliyun.dms_enterprise20181101.Client createDmsClient() throws Exception {
        if (dmsClient != null) {
            return dmsClient;
        }
        com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
                .setCredential(getCredentialClient());
        // For the endpoint, see https://api.aliyun.com/product/dms-enterprise
        config.endpoint = "dms-enterprise.cn-hangzhou.aliyuncs.com";
        dmsClient = new com.aliyun.dms_enterprise20181101.Client(config);
        return dmsClient;
    }

    // Create an SLS client to call PutLogs.
    public static synchronized com.aliyun.openservices.log.Client createSlsClient() throws Exception {
        if (slsClient != null) {
            return slsClient;
        }
        com.aliyun.credentials.Client credentialClient = getCredentialClient();
        // For the endpoint, see https://api.aliyun.com/product/Sls
        String endpoint = "cn-hangzhou.log.aliyuncs.com";
        slsClient = new com.aliyun.openservices.log.Client(endpoint, credentialClient.getAccessKeyId(), credentialClient.getAccessKeySecret());
        return slsClient;
    }

    // Call GetOpLog to retrieve DMS operation logs for the specified time range and page.
    public static List<GetOpLogResponseBody.GetOpLogResponseBodyOpLogDetailsOpLogDetail> getLogs(com.aliyun.dms_enterprise20181101.Client client, String startTime, String endTime, Integer pageNumber, Integer pageSize) {
        com.aliyun.dms_enterprise20181101.models.GetOpLogRequest getOpLogRequest = new com.aliyun.dms_enterprise20181101.models.GetOpLogRequest()
                .setStartTime(startTime)
                .setEndTime(endTime)
                .setPageSize(pageSize)
                .setPageNumber(pageNumber);
        com.aliyun.teautil.models.RuntimeOptions runtime = new com.aliyun.teautil.models.RuntimeOptions();
        try {
            GetOpLogResponse response = client.getOpLogWithOptions(getOpLogRequest, runtime);
            return Optional.ofNullable(response.getBody())
                    .map(GetOpLogResponseBody::getOpLogDetails)
                    .map(GetOpLogResponseBody.GetOpLogResponseBodyOpLogDetails::getOpLogDetail)
                    .orElse(new ArrayList<>());
        } catch (Exception e) {
            System.out.println(e.getMessage());
            throw new RuntimeException(e.getMessage());
        }
    }

    // Call PutLogs to write logs to the SLS Logstore.
    public static void putLogs(com.aliyun.openservices.log.Client client, String project, String logStore, List<GetOpLogResponseBody.GetOpLogResponseBodyOpLogDetailsOpLogDetail> logDetailList) {
        List<LogItem> logItemList = new ArrayList<>();
        for (GetOpLogResponseBody.GetOpLogResponseBodyOpLogDetailsOpLogDetail logDetail : logDetailList) {
            LogItem logItem = new LogItem((int) (new Date().getTime() / 1000));
            logItem.PushBack("module", logDetail.getModule());
            logItem.PushBack("database", logDetail.getDatabase());
            logItem.PushBack("userId", logDetail.getUserId());
            logItem.PushBack("opUserId", String.valueOf(logDetail.getOpUserId()));
            logItem.PushBack("userNick", logDetail.getUserNick());
            logItem.PushBack("opTime", logDetail.getOpTime());
            logItem.PushBack("opContent", logDetail.getOpContent());
            logItem.PushBack("orderId", String.valueOf(logDetail.getOrderId()));
            logItemList.add(logItem);
        }
        try {
            client.PutLogs(project, logStore, "", logItemList, "");
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println(e.getMessage());
            throw new RuntimeException(e.getMessage());
        }
    }

    // Create an index. You only need to initialize a Logstore once.
    public static void createIndex(com.aliyun.openservices.log.Client client, String project, String logStore) {
        try {
            GetIndexResponse getIndexResponse = client.GetIndex(project, logStore);
            Index index = getIndexResponse.GetIndex();
            IndexLine indexLine = index.GetLine();
            List<String> indexToken = null;
            if (indexLine != null) {
                indexToken = indexLine.GetToken();
            }
            IndexKeys keys = new IndexKeys();
            List<String> logStoreKeys = List.of("module", "database", "userId", "opUserId",
                    "userNick", "opTime", "opContent", "orderId");
            for (String logStoreKey : logStoreKeys) {
                IndexKey key = new IndexKey();
                key.SetType("text");
                key.SetChn(true);
                key.SetDocValue(false);
                key.SetToken(indexToken);
                keys.AddKey(logStoreKey, key);
                index.SetKeys(keys);
            }
            client.UpdateIndex(project, logStore, index);
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println(e.getMessage());
            throw new RuntimeException(e.getMessage());
        }
    }

    public static void main(String[] args) throws Exception {
        // The start and end times for the DMS operation logs (accurate to the second).
        String startStr = "2025-01-10 14:30:00";
        String endStr = "2025-01-13 09:15:00";
        // The SLS project and Logstore names.
        String project = "project";
        String logStore = "logStore";

        // Initialize the Logstore index. Only needs to run once per Logstore.
        createIndex(createSlsClient(), project, logStore);

        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        LocalDateTime start = LocalDateTime.parse(startStr, formatter);
        LocalDateTime end = LocalDateTime.parse(endStr, formatter);

        if (start.isAfter(end)) {
            throw new IllegalArgumentException("The start time cannot be later than the end time.");
        }

        LocalDateTime current = start;

        // DMS log volumes can be large. To avoid API timeouts, the code fetches logs one day at a time.
        while (!current.isAfter(end)) {
            // For each day, use either 23:59:59 or the overall end time, whichever comes first.
            LocalDateTime dayEnd = current.toLocalDate().atTime(23, 59, 59);
            if (dayEnd.isAfter(end)) {
                dayEnd = end;
            }

            String segmentStartStr = current.format(formatter);
            String segmentEndStr = dayEnd.format(formatter);

            // Fetch logs from DMS and write them to SLS.
            process(segmentStartStr, segmentEndStr, project, logStore);

            // Advance to the start of the next day.
            current = dayEnd.toLocalDate().plusDays(1).atStartOfDay();
        }
    }

    public static void process(String startTime, String endTime, String project, String logStore) throws Exception {
        int pageNumber = 1;
        int pageSize = 100;
        com.aliyun.dms_enterprise20181101.Client dmsClient = createDmsClient();
        com.aliyun.openservices.log.Client slsClient = createSlsClient();
        List<GetOpLogResponseBody.GetOpLogResponseBodyOpLogDetailsOpLogDetail> logs = getLogs(dmsClient, startTime, endTime, pageNumber, pageSize);
        while(logs.size() >= 100) {
            putLogs(slsClient, project, logStore, logs);
            pageNumber++;
            logs = getLogs(dmsClient, startTime, endTime, pageNumber, pageSize);
        }
    }
}

What's next

After exporting DMS operation logs to SLS, query and analyze them. For more information, see Query and analyze DMS operation logs in Simple Log Service.