Object Storage Service:Create a data pipeline (Java SDK V2)

Prerequisites

• OSS Java SDK V2 is installed. If you use Maven, add the com.aliyun:alibabacloud-oss-v2 dependency to your pom.xml.

• Access credentials are configured by using environment variables. The example in this topic uses EnvironmentVariableCredentialsProvider to read credentials from the OSS_ACCESS_KEY_ID and OSS_ACCESS_KEY_SECRET environment variables.

• DataPipeline operations are forwarded by OSS to the underlying workflow service. Make sure that the source standard bucket and the destination vector bucket are located in the same region.

Sample code

import com.aliyun.sdk.service.oss2.credentials.CredentialsProvider;
import com.aliyun.sdk.service.oss2.credentials.EnvironmentVariableCredentialsProvider;
import com.aliyun.sdk.service.oss2.dataprocess.OSSDataProcessClient;
import com.aliyun.sdk.service.oss2.dataprocess.models.*;
import java.util.Arrays;
import java.util.Collections;

public class PutDataPipelineConfigurationSample {
    public static void main(String[] args) throws Exception {
        String role = "acs:ram::<AccountId>:role/my-data-pipeline-role";
        CredentialsProvider provider = new EnvironmentVariableCredentialsProvider();
        try (OSSDataProcessClient client = OSSDataProcessClient.newBuilder()
                .region("cn-hangzhou")
                .credentialsProvider(provider)
                .build()) {

            DataPipelineSourceFilterConfiguration filterConfig =
                    DataPipelineSourceFilterConfiguration.newBuilder()
                            .prefixSet(Arrays.asList("data/"))
                            .objectMediaTypes(Arrays.asList("image", "video"))
                            .build();

            DataPipelineSource source = DataPipelineSource.newBuilder()
                    .inputBucket("my-bucket")
                    .inputDataScope("All")
                    .ignoreDelete(true)
                    .filterConfiguration(filterConfig)
                    .build();

            DataPipelineEmbeddingConfiguration embeddingConfig =
                    DataPipelineEmbeddingConfiguration.newBuilder()
                            .embeddingProvider("bailian")
                            .apiKey("sk-xxxx")
                            .model("qwen2.5-vl-embedding")
                            .fps(1.0f)
                            .build();

            DataPipelineDestination destination = DataPipelineDestination.newBuilder()
                    .vectorBucketName("my-vector-bucket")
                    .vectorIndexNames(Collections.singletonList("my-index"))
                    .build();

            PutDataPipelineConfigurationConfiguration config =
                    PutDataPipelineConfigurationConfiguration.newBuilder()
                            .dataPipelineDescription("Data pipeline sample")
                            .sources(Collections.singletonList(source))
                            .dataPipelineEmbeddingConfiguration(embeddingConfig)
                            .destination(destination)
                            .build();

            PutDataPipelineConfigurationResult result = client.putDataPipelineConfiguration(
                    PutDataPipelineConfigurationRequest.newBuilder()
                            .dataPipelineName("my-data-pipeline")
                            .role(role)
                            .putDataPipelineConfigurationConfiguration(config)
                            .build());

            System.out.printf("Status code:%d, request id:%s%n",
                    result.statusCode(), result.requestId());

        } catch (Exception e) {
            System.out.printf("error:%n%s", e);
        }
    }
}

References

• For the synchronous sample code, see PutDataPipelineConfiguration.java.

• For the asynchronous sample code, see PutDataPipelineConfigurationAsync.java.