AI_PII_MASK 函数用于识别并掩码文本中的个人敏感信息,适用于客服日志、工单归档和分析数据的隐私保护。
命令格式
REST 接口
POST /v2/vectordb/ai/pii_mask
Content-Type: application/json
{
"model_name": "<模型名>",
"texts": ["<文本>"],
"params": {"pii_types": ["EMAIL", "PHONE"]}
}
Python
schema = MilvusClient.create_schema(auto_id=True, enable_dynamic_field=False)
schema.add_field("id", DataType.INT64, is_primary=True)
schema.add_field("content", DataType.VARCHAR, max_length=4096)
schema.add_field("masked", DataType.VARCHAR, max_length=4096)
schema.add_field("dummy_vector", DataType.FLOAT_VECTOR, dim=2)
schema.add_function(
Function(
name="mask_pii",
function_type=texttransform_function_type(),
input_field_names=["content"],
output_field_names=["masked"],
params={
"provider": "aliyun_milvus",
"model_name": "<模型名>",
"task": "ai_pii_mask",
"pii_types": "PERSON,EMAIL,PHONE",
"mask_char": "*",
"preserve_length": "true",
"temperature": "0",
},
)
)
参数说明
|
参数 |
说明 |
|
|
必填。已配置的文本模型名称。 |
|
|
REST 必填。待脱敏文本数组。 |
|
|
选填。要脱敏的类型,支持数组或逗号分隔字符串,数量 1~30;默认覆盖姓名、邮箱、电话、证件号、银行卡、地址、URL、IP、令牌和密码等。 |
|
|
选填。单个替换字符,默认为 |
|
|
选填。是否保持原敏感片段长度,默认 |
|
|
选填。模型稳定性、并发和超时设置。 |
|
|
仅 Collection Function 必填,固定为 |
返回值说明
data.output.outputs 返回与输入同序的脱敏文本。Collection Function 会将结果写入目标文本字段。
使用示例:客服日志入库前脱敏
运营需要保留联系记录以便统计,但不能在分析库中存储姓名、邮箱和电话。下面用 REST 脱敏一条日志,并用 Collection Function 在写入时自动脱敏。
REST 接口
#!/usr/bin/env bash
set -euo pipefail
MILVUS_REST_BASE_URL="http://c-xxxx.milvus.aliyuncs.com:19530"
MILVUS_AUTH_TOKEN="<yourUsername>:<yourPassword>"
post_json() {
local path="$1"
local body="$2"
curl -X POST \
"$MILVUS_REST_BASE_URL$path" \
-H "Authorization: Bearer $MILVUS_AUTH_TOKEN" \
-H "Content-Type: application/json" \
-d "$body"
}
MODEL_NAME="qwen3.7-max"
BODY=$(cat <<JSON
{
"model_name": "$MODEL_NAME",
"texts": ["Please contact Zhang San at zhangsan@example.com or +86 13812345678."],
"params": {
"pii_types": ["PERSON", "EMAIL", "PHONE"],
"mask_char": "*",
"preserve_length": true,
"temperature": 0
}
}
JSON
)
RESPONSE_BODY="$(post_json "/v2/vectordb/ai/pii_mask" "$BODY")"
if command -v jq >/dev/null 2>&1; then
echo "$RESPONSE_BODY" | jq .
[ "$(echo "$RESPONSE_BODY" | jq -r '.code // -1')" = "0" ] || exit 1
else
echo "$RESPONSE_BODY"
fi
# 预期:masked = Please contact ********* at ******************** or ***************.
Python
安装 PyMilvus 后,将 MILVUS_URI、MILVUS_TOKEN 占位值替换为实际集群地址和 token 后运行;示例已内联公共辅助函数。
from __future__ import annotations
from typing import Any
from pymilvus import DataType, Function, FunctionType, MilvusClient
MILVUS_URI = "http://c-xxxx.milvus.aliyuncs.com:19530"
MILVUS_TOKEN = "<yourUsername>:<yourPassword>"
DUMMY_VECTOR_DIM = 2
TEXTTRANSFORM_FUNCTION_TYPE = 9
def texttransform_function_type() -> Any:
for type_name in ("TEXTTRANSFORM", "TEXT_TRANSFORM", "TextTransform"):
function_type = getattr(FunctionType, type_name, None)
if function_type is not None:
return function_type
# 阿里云 Milvus 将 TEXTTRANSFORM 作为托管扩展(函数类型值 9)提供;
# 部分 pymilvus 版本尚未内置该枚举成员,而 Function(...) 通过 FunctionType(...) 校验。
existing = getattr(FunctionType, "_value2member_map_", {}).get(TEXTTRANSFORM_FUNCTION_TYPE)
if existing is not None:
return existing
extension = int.__new__(FunctionType, TEXTTRANSFORM_FUNCTION_TYPE)
extension._name_ = "TEXTTRANSFORM"
extension._value_ = TEXTTRANSFORM_FUNCTION_TYPE
FunctionType._value2member_map_[TEXTTRANSFORM_FUNCTION_TYPE] = extension
FunctionType._member_map_["TEXTTRANSFORM"] = extension
return extension
def add_id(schema: Any) -> None:
schema.add_field("id", DataType.INT64, is_primary=True)
def add_dummy_vector(schema: Any) -> None:
schema.add_field("dummy_vector", DataType.FLOAT_VECTOR, dim=DUMMY_VECTOR_DIM)
def run_texttransform_example(*, client, collection_name, input_fields, output_field, function_name, function_params, rows) -> None:
if client.has_collection(collection_name):
client.drop_collection(collection_name)
schema = MilvusClient.create_schema(auto_id=True, enable_dynamic_field=False)
add_id(schema)
for name, data_type, max_length in input_fields:
field_params = {"max_length": max_length} if max_length is not None else {}
schema.add_field(name, data_type, **field_params)
output_name, output_data_type, output_max_length = output_field
output_params = {"max_length": output_max_length} if output_max_length is not None else {}
schema.add_field(output_name, output_data_type, **output_params)
add_dummy_vector(schema)
schema.add_function(
Function(
name=function_name,
function_type=texttransform_function_type(),
input_field_names=[name for name, _, _ in input_fields],
output_field_names=[output_name],
params=function_params,
)
)
index_params = client.prepare_index_params()
index_params.add_index(field_name="dummy_vector", index_type="AUTOINDEX", metric_type="COSINE")
client.create_collection(collection_name=collection_name, schema=schema, index_params=index_params)
client.insert(collection_name, rows)
client.flush(collection_name)
fields = [name for name, _, _ in input_fields] + [output_name]
for row in client.query(collection_name, filter="", output_fields=fields, limit=len(rows)):
print(row)
MODEL_NAME = "qwen3.7-max"
client = MilvusClient(uri=MILVUS_URI, token=MILVUS_TOKEN)
run_texttransform_example(
client=client,
collection_name="simple_ai_pii_mask",
input_fields=[("content", DataType.VARCHAR, 4096)],
output_field=("masked", DataType.VARCHAR, 4096),
function_name="mask_pii",
function_params={"provider": "aliyun_milvus", "model_name": MODEL_NAME, "task": "ai_pii_mask", "pii_types": "PERSON,EMAIL,PHONE", "mask_char": "*", "preserve_length": "true", "temperature": "0"},
rows=[{"content": "Please contact Zhang San at zhangsan@example.com or +86 13812345678.", "dummy_vector": [0.1, 0.2]}],
)
# 预期:masked 得到 Please contact ********* at ******************** or ***************.
预期结果:masked 得到 Please contact ********* at ******************** or ***************.,姓名、邮箱、电话被掩码且长度保持,可安全写入分析集合。