Simple Log Service:Consume log data

Prerequisites​

• A Resource Access Management (RAM) user is created, and the required permissions are granted to the RAM user. For more information, see Create a RAM user and authorize the RAM user to access Simple Log Service.

• The ALIBABA_CLOUD_ACCESS_KEY_ID and ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variables are configured. For more information, see Configure environment variables in Linux, macOS, and Windows.

  Important

  • The AccessKey pair of an Alibaba Cloud account has permissions on all API operations. We recommend that you use the AccessKey pair of a RAM user to call API operations or perform routine O&M.

  • We recommend that you do not include your AccessKey ID or AccessKey secret in your project code. Otherwise, the AccessKey pair may be leaked and the security of all resources within your account may be compromised.

• An SDK development environment is set up. For more information, see Overview of Simple Log Service SDKs.

Prerequisites​

• A Resource Access Management (RAM) user is created, and the required permissions are granted to the RAM user. For more information, see Create a RAM user and authorize the RAM user to access Simple Log Service.

• The ALIBABA_CLOUD_ACCESS_KEY_ID and ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variables are configured. For more information, see Configure environment variables in Linux, macOS, and Windows.

  Important

  • The AccessKey pair of an Alibaba Cloud account has permissions on all API operations. We recommend that you use the AccessKey pair of a RAM user to call API operations or perform routine O&M.

  • We recommend that you do not include your AccessKey ID or AccessKey secret in your project code. Otherwise, the AccessKey pair may be leaked and the security of all resources within your account may be compromised.

• An SDK development environment is set up. For more information, see Overview of Simple Log Service SDKs.

Procedure

The following example shows how to call the PullLog operation to obtain log data and use Simple Log Service SDK for Java to perform SPL-based consumption. For more information about Simple Log Service SDK, see Overview of Simple Log Service SDK.

1. Add Maven dependencies. Open the pom.xml file in the root directory of your Java project and add the following code:

   <dependency>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-java</artifactId>
     <version>2.5.0</version>
   </dependency>
   <dependency>
   <groupId>com.aliyun.openservices</groupId>
     <artifactId>aliyun-log</artifactId>
     <version>0.6.99</version>
   </dependency>

2. Create a file named PullLogsWithSPLDemo.java.

   import com.aliyun.openservices.log.Client;
   import com.aliyun.openservices.log.common.*;
   import com.aliyun.openservices.log.common.Consts;
   import com.aliyun.openservices.log.exception.LogException;
   import com.aliyun.openservices.log.request.PullLogsRequest;
   import com.aliyun.openservices.log.response.ListShardResponse;
   import com.aliyun.openservices.log.response.PullLogsResponse;
   
   import java.util.HashMap;
   import java.util.List;
   import java.util.Map;
   
   public class PullLogsWithSPLDemo {
       // The Simple Log Service endpoint. In this example, the Simple Log Service endpoint for the China (Hangzhou) region is used. Replace the parameter value with the actual endpoint.
       private static final String endpoint = "cn-hangzhou.log.aliyuncs.com";
       // In this example, the AccessKey ID and AccessKey secret are obtained from environment variables. 
       private static final String accessKeyId = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID");
       private static final String accessKeySecret = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET");
       // The name of the project.
       private static final String project = "your_project";
       // The name of the Logstore.
       private static final String logStore = "your_logstore";
   
       public static void main(String[] args) throws Exception {
           // Create a client for Simple Log Service.
           Client client = new Client(endpoint, accessKeyId, accessKeySecret);
           // Query the shards of the Logstore.
           ListShardResponse resp = client.ListShard(project, logStore);
           System.out.printf("%s has %d shards\n", logStore, resp.GetShards().size());
           Map<Integer, String> cursorMap = new HashMap<Integer, String>();
           for (Shard shard : resp.GetShards()) {
               int shardId = shard.getShardId();
               // Use the BEGIN cursor or obtain a specific cursor to consume log data. If you want to use the END cursor to consume log data, use Consts.CursorMode.END.
               cursorMap.put(shardId, client.GetCursor(project, logStore, shardId, Consts.CursorMode.BEGIN).GetCursor());
           }
           try {
               while (true) {
                   // Obtain logs from each shard.
                   for (Shard shard : resp.GetShards()) {
                       int shardId = shard.getShardId();
                       PullLogsRequest request = new PullLogsRequest(project, logStore, shardId, 1000, cursorMap.get(shardId));
                       request.setQuery("* | where cast(body_bytes_sent as bigint) > 14000");
                       request.setPullMode("scan_on_stream");
                       PullLogsResponse response = client.pullLogs(request);
                       // Split log groups into logs by logic. 
                       List<LogGroupData> logGroups = response.getLogGroups();
                       System.out.printf("Get %d logGroup from logStore:%s:\tShard:%d\n", logGroups.size(), logStore, shardId);
   
                       // Move the cursor after the obtained logs are processed. 
                       cursorMap.put(shardId, response.getNextCursor());
                   }
               }
           } catch (LogException e) {
               System.out.println("error code :" + e.GetErrorCode());
               System.out.println("error message :" + e.GetErrorMessage());
               throw e;
           }
       }
   }