Tablestore:Optimalkan kinerja pengambilan vektor di Tablestore

Rumus penilaian pengambilan vektor

Pengambilan vektor mendukung tiga ukuran jarak: Jarak Euclidean (euclidean), kemiripan kosinus (cosine), dan hasil kali titik (dot_product). Setiap ukuran jarak memiliki rumus penilaian yang berbeda, yang digunakan Tablestore untuk mengevaluasi kemiripan antar vektor. Tabel berikut menjelaskan rumus-rumus tersebut.

Pemecahan masalah dan analisis

private static void knnVectorQuery(SyncClient client) {
    SearchQuery searchQuery = new SearchQuery();
    KnnVectorQuery query = new KnnVectorQuery();
    query.setFieldName("Col_Vector");
    query.setTopK(10); // Mengembalikan K tetangga terdekat teratas.
    query.setFloat32QueryVector(new float[]{0.1f, 0.2f, 0.3f, 0.4f});
    // Vektor tetangga terdekat harus memenuhi kondisi: Col_Keyword=hangzhou && Col_Long<4.
    query.setFilter(QueryBuilders.bool()
            .must(QueryBuilders.term("Col_Keyword", "hangzhou"))
            .must(QueryBuilders.range("Col_Long").lessThan(4))
    );
    searchQuery.setQuery(query);
    searchQuery.setLimit(10);
    // Urutkan berdasarkan skor.
    searchQuery.setSort(new Sort(Collections.singletonList(new ScoreSort())));
    SearchRequest searchRequest = new SearchRequest("<TABLE_NAME>", "<SEARCH_INDEX_NAME>", searchQuery);
    SearchRequest.ColumnsToGet columnsToGet = new SearchRequest.ColumnsToGet();
    columnsToGet.setColumns(Arrays.asList("Col_Keyword", "Col_Long"));
    searchRequest.setColumnsToGet(columnsToGet);
    // Akses antarmuka Search.
    SearchResponse resp = client.search(searchRequest);
    for (SearchHit hit : resp.getSearchHits()) {
        // Cetak skor.
        System.out.println(hit.getScore());
        // Cetak data.
        System.out.println(hit.getRow());
    }
}

Lampiran: Kode demo untuk rumus penilaian pengambilan vektor

Kode Java berikut menunjukkan rumus penilaian untuk ukuran jarak yang didukung.

import java.util.concurrent.ThreadLocalRandom;

public class CompareVector {

    public static void main(String[] args) {
        // a adalah vektor kueri
        float[] a = randomVector(512);
        // b adalah vektor dari baris dalam indeks yang Anda harapkan dikembalikan
        float[] b = randomVector(512);
        // Pilih algoritma ukuran kemiripan yang Anda atur dalam indeks pencarian dan keluarkan skornya
        System.out.println(MetricFunction.COSINE.compare(a, b));
    }

    public static float[] randomVector(int dim) {
        float[] vec = new float[dim];
        for (int i = 0; i < dim; i++) {
            vec[i] = ThreadLocalRandom.current().nextFloat();
            if (ThreadLocalRandom.current().nextBoolean()) {
                vec[i] = -vec[i];
            }
        }
        return l2normalize(vec, true);
    }

    public static float[] l2normalize(float[] v, boolean throwOnZero) {
        double squareSum = 0.0f;
        int dim = v.length;
        for (float x : v) {
            squareSum += x * x;
        }
        if (squareSum == 0) {
            if (throwOnZero) {
                throw new IllegalArgumentException("normalize a zero-length vector");
            } else {
                return v;
            }
        }
        double length = Math.sqrt(squareSum);
        for (int i = 0; i < dim; i++) {
            v[i] /= length;
        }
        return v;
    }

    public enum MetricFunction {
        /**
         * Jarak Euclidean.
         */
        EUCLIDEAN {
            @Override
            public float compare(float[] v1, float[] v2) {
                return 1 / (1 + VectorUtil.squareDistance(v1, v2));
            }
        },

        /**
         * Hasil kali titik.
         */
        DOT_PRODUCT {
            @Override
            public float compare(float[] v1, float[] v2) {
                return (1 + VectorUtil.dotProduct(v1, v2)) / 2;
            }
        },

        /**
         * Kosinus.
         */
        COSINE {
            @Override
            public float compare(float[] v1, float[] v2) {
                return (1 + VectorUtil.cosine(v1, v2)) / 2;
            }
        };

        public abstract float compare(float[] v1, float[] v2);
    }


    static final class VectorUtil {

        private static void checkParam(float[] a, float[] b) {
            if (a.length != b.length) {
                throw new IllegalArgumentException("vector dimensions differ: " + a.length + "!=" + b.length);
            }
        }

        public static float dotProduct(float[] a, float[] b) {
            checkParam(a, b);
            float res = 0f;
            for (int i = 0; i < a.length; i++) {
                res += b[i] * a[i];
            }
            return res;
        }

        public static float cosine(float[] a, float[] b) {
            checkParam(a, b);
            float sum = 0.0f;
            float norm1 = 0.0f;
            float norm2 = 0.0f;
            for (int i = 0; i < a.length; i++) {
                float elem1 = a[i];
                float elem2 = b[i];
                sum += elem1 * elem2;
                norm1 += elem1 * elem1;
                norm2 += elem2 * elem2;
            }
            return (float) (sum / Math.sqrt((double) norm1 * (double) norm2));
        }

        public static float squareDistance(float[] a, float[] b) {
            checkParam(a, b);
            float sum = 0.0f;
            for (int i = 0; i < a.length; i++) {
                float difference = a[i] - b[i];
                sum += difference * difference;
            }
            return sum;
        }
    }
}