MaxCompute:Vector index

Create a vector index

Create an index on a vector column to accelerate retrieval queries. The command is as follows:

CREATE VECTOR INDEX <index_name> 
ON <table_name> (<description_embedding>)
IDXPROPERTIES (
  'algorithm' = 'xxx', 
  'distance_type' = 'xxx', 
  'build_params' = 'xxx'
  ...
);

Parameters

Configure vector index quantization

Vector indexes support internal vector quantization to reduce index storage, runtime memory, and memory access overhead during retrieval. Quantization changes only the vector encoding within the index. It does not alter the original VECTOR(FLOAT, dimension) data in the table or how VECTOR_SEARCH is called. MaxCompute vector indexes support the following two retrieval methods:

• Single-stage retrieval: Directly uses the base_quantization_type for graph traversal and distance calculation. This method results in a simpler index structure and lower memory usage.

• Two-stage retrieval: First uses low-precision base vectors for fast candidate retrieval, and then uses precise vectors to recalculate distances. This method strikes a balance between retrieval performance and recall.

The following table describes common combinations of quantization parameters.

Quantization provides the following benefits:

• Reduces the index file size and runtime memory footprint.

• Lowers the memory bandwidth overhead during index loading and retrieval.

• Improves CPU cache utilization, which allows a single worker to load and process more vectors.

• Lets you balance retrieval performance, resource costs, and recall by combining base quantization with high-precision reordering.

We recommend using FP16 when creating an HGraph index. For more examples, see Example: Create a vector index with FP16 quantization and Example: Create a vector index with RaBitQ low-precision traversal and high-precision reordering.

Populate data

INSERT OVERWRITE TABLE <table_name> [PARTITION <partition_spec>]
SELECT ......

Prepare data

SET odps.sql.type.system.odps2=true;
SET odps.sql.type.vector.enable=true;

DROP TABLE IF EXISTS vector_test;
DROP VIEW IF EXISTS vector_test;

CREATE TABLE IF NOT EXISTS vector_test(
  c0 int,
  c1 vector(float, 2),
  c2 vector(float, 3)
) STORED AS aliorc 
TBLPROPERTIES (
  'table.format.version'='2', 
  'acid.data.retain.hours'='24',
  'columnar.nested.type'='true', 
  'transactional'='true'
);


INSERT OVERWRITE vector_test SELECT 1, vector(1.1F,2.2F), vector(1.1F,2.2F,3.3F) UNION ALL 
SELECT 2, vector(2.2F,3.3F), vector(2.2F,3.3F,4.4F) 
UNION ALL 
SELECT 3, vector(3.3F,4.4F), vector(3.3F,4.4F,5.5F);

Example: Create a vector index and insert data

CREATE VECTOR INDEX c2_vector_index 
ON vector_test (c2) 
IDXPROPERTIES (
  'algorithm' = 'hgraph', 
  'distance_type' = 'cosine', 
  'build_params' = '{"max_degree": 16, "ef_construction": 128}');


ALTER TABLE vector_test REBUILD INDEX c2_vector_index; 


INSERT OVERWRITE vector_test SELECT 1, vector(1.1F,2.2F), vector(1.1F,2.2F,3.3F) 
UNION ALL  
SELECT 2, vector(2.2F,3.3F), vector(2.2F,3.3F,4.4F) 
UNION ALL 
SELECT 3, vector(3.3F,4.4F), vector(3.3F,4.4F,5.5F) 
UNION ALL 
SELECT 4, vector(4.4F,5.5F), vector(4.4F,5.5F,6.6F) 
UNION ALL 
SELECT 5, vector(5.5F,6.6F), vector(5.5F,6.6F,7.7F); 

Example: Rebuild a vector index

ALTER TABLE vector_test REBUILD INDEX c2_vector_index; 

SELECT * FROM vector_test; 
-- The following result is returned:
+------+------+------+
| c0   | c1   | c2   |
+------+------+------+
| 1    | [1.1, 2.2] | [1.1, 2.2, 3.3] |
| 2    | [2.2, 3.3] | [2.2, 3.3, 4.4] |
| 3    | [3.3, 4.4] | [3.3, 4.4, 5.5] |
| 4    | [4.4, 5.5] | [4.4, 5.5, 6.6] |
| 5    | [5.5, 6.6] | [5.5, 6.6, 7.7] |
+------+------+------+

Example: List vector indexes

SHOW INDEXES ON vector_test;
-- The following result is returned:
{"Indexes": [{
            "createTime": 1779900024105,
            "id": "512d520**45e1c7ba8",
            "indexColumns": [{"name": "c2"}],
            "name": "c2_vector_index",
            "properties": {
                "algorithm": "hgraph",
                "build_params": "{\"max_degree\": 16, \"ef_construction\": 128}",
                "distance_type": "cosine"},
            "type": "VECTOR"}]}

Example: View vector index information

DESC INDEX c2_vector_index ON vector_test;

-- The following result is returned:
+------------------------------------------------------------------------------------+
| Index Detail                                                                       |
+------------------------------------------------------------------------------------+
| name:                     c2_vector_index                                          |
| id:                       512d520**e1c7ba8                         |
| index_type:               VECTOR                                                   |
| index_columns:            c2                                                       |
| status:                   ACTIVE                                                   |
| coverage_percentage:      100%                                                     |
| storage_size_bytes:       6940                                                     |
| properties:               build_params={"max_degree": 16, "ef_construction": 128}, distance_type=cosine, algorithm=hgraph |
+------------------------------------------------------------------------------------+

Example: Drop a vector index

DROP INDEX c2_vector_index ON vector_test;

-- The following result is returned:
{"Indexes": []}

Example: Create a vector index with FP16 quantization

CREATE VECTOR INDEX doc_vector_index
ON doc_table (embedding)
IDXPROPERTIES (
  'algorithm' = 'hgraph',
  'distance_type' = 'dot_product',
  'build_params' = '{
    "max_degree":48,
    "ef_construction":400,
    "base_quantization_type":"fp16"
  }'
);

-- After you create the index, you must rebuild it for existing data.
ALTER TABLE doc_table PARTITION (pt='20260730') REBUILD INDEX doc_vector_index;

Example: Create a vector index with RaBitQ and reordering

CREATE VECTOR INDEX doc_vector_index
ON doc_table (embedding)
IDXPROPERTIES (
  'algorithm' = 'hgraph',
  'distance_type' = 'dot_product',
  'build_params' ='{
    "max_degree":48,
    "ef_construction":400,
    "base_quantization_type":"rabitq",
    "use_reorder":true,
    "precise_quantization_type":"fp32"
  }'
);