Platform For AI:TensorFlow processor

Prerequisites

Before you begin, ensure that you have:

• A TensorFlow model in SavedModel format. Keras and checkpoint models must be converted before deployment. For conversion steps, see Export TensorFlow models in the SavedModel format.

• (Optional) A model optimized by PAI-Blade. PAI-Blade-optimized models run directly without conversion.

Deploy a model

{
  "name": "tf_serving_test",
  "model_path": "http://examplebucket.oss-cn-shanghai.aliyuncs.com/models/model.tar.gz",
  "processor": "tensorflow_cpu_1.15",
  "metadata": {
    "instance": 1,
    "cpu": 1,
    "memory": 4000
  }
}

Call a service

TensorFlow services use protobuf format for both input and output. Online debugging does not support protobuf format.

EAS provides a software development kit (SDK) that handles request serialization and includes direct connection and fault tolerance mechanisms. Use the SDK unless you need to build raw protobuf requests.

# Empty request — returns model structure
curl <endpoint>/api/predict/<service-name> \
  -H 'Authorization: <authentication-token>'

{
    "inputs": [
        {
            "name": "images",
            "shape": [-1, 784],
            "type": "DT_FLOAT"
        }
    ],
    "outputs": [
        {
            "name": "scores",
            "shape": [-1, 10],
            "type": "DT_FLOAT"
        }
    ],
    "signature_name": "predict_images"
}

#!/usr/bin/env python

from eas_prediction import PredictClient
from eas_prediction import TFRequest

if __name__ == '__main__':
    # Initialize client with endpoint and service name
    client = PredictClient('http://1828488879222***.cn-shanghai.pai-eas.aliyuncs.com', 'mnist_saved_model_example')
    client.set_token('YTg2ZjE0ZjM4ZmE3OTc0NzYxZDMyNmYzMTJjZTQ1****')
    client.init()

    # Build request with signature name
    req = TFRequest('predict_images')
    req.add_feed('images', [1, 784], TFRequest.DT_FLOAT, [1] * 784)

    # Send requests
    for x in range(0, 1000000):
        resp = client.predict(req)
        print(resp)

Configure model warm-up

TensorFlow models use deferred initialization: when a model first receives a request, it loads files and parameters into memory. This initialization can cause the first requests to fail with errors such as 408 (timeout) or 450 (queue full), and produces latency spikes during rolling updates.

Configure warm-up to prevent service jitter during rolling updates.

{
  "name": "tf_serving_test",
  "model_path": "http://examplebucket.oss-cn-shanghai.aliyuncs.com/models/model.tar.gz",
  "processor": "tensorflow_cpu_1.15",
  "warm_up_data_path": "oss://path/to/warm_up_test.bin",
  "metadata": {
    "instance": 1,
    "cpu": 1,
    "memory": 4000
  }
}

Request format

The SDK handles request serialization automatically. To build requests manually, generate code from the protobuf definition below. For a complete walkthrough, see Construct a TensorFlow service request.

syntax = "proto3";
option cc_enable_arenas = true;
option java_package = "com.aliyun.openservices.eas.predict.proto";
option java_outer_classname = "PredictProtos";

enum ArrayDataType {
  DT_INVALID = 0;
  DT_FLOAT = 1;
  DT_DOUBLE = 2;
  DT_INT32 = 3;
  DT_UINT8 = 4;
  DT_INT16 = 5;
  DT_INT8 = 6;
  DT_STRING = 7;
  DT_COMPLEX64 = 8;   // Single-precision complex
  DT_INT64 = 9;
  DT_BOOL = 10;
  DT_QINT8 = 11;      // Quantized int8
  DT_QUINT8 = 12;     // Quantized uint8
  DT_QINT32 = 13;     // Quantized int32
  DT_BFLOAT16 = 14;   // Float32 truncated to 16 bits (cast ops only)
  DT_QINT16 = 15;     // Quantized int16
  DT_QUINT16 = 16;    // Quantized uint16
  DT_UINT16 = 17;
  DT_COMPLEX128 = 18; // Double-precision complex
  DT_HALF = 19;
  DT_RESOURCE = 20;
  DT_VARIANT = 21;    // Arbitrary C++ data types
}

// Dimensions of an array
message ArrayShape {
  repeated int64 dim = 1 [packed = true];
}

// Protocol buffer representing an array
message ArrayProto {
  ArrayDataType dtype = 1;
  ArrayShape array_shape = 2;
  repeated float float_val = 3 [packed = true];    // DT_FLOAT
  repeated double double_val = 4 [packed = true];  // DT_DOUBLE
  repeated int32 int_val = 5 [packed = true];      // DT_INT32, DT_INT16, DT_INT8, DT_UINT8
  repeated bytes string_val = 6;                   // DT_STRING
  repeated int64 int64_val = 7 [packed = true];    // DT_INT64
  repeated bool bool_val = 8 [packed = true];      // DT_BOOL
}

// Specifies the model signature and input tensors; filters outputs before returning
message PredictRequest {
  string signature_name = 1;
  map<string, ArrayProto> inputs = 2;
  repeated string output_filter = 3;
}

// Contains output tensors from a successful prediction
message PredictResponse {
  map<string, ArrayProto> outputs = 1;
}

Processor versions

Use the latest processor version unless you have a specific version requirement. Newer versions provide better performance and forward compatibility.

What's next

• Export TensorFlow models in the SavedModel format

• Using the Python SDK

• Construct a TensorFlow service request

• Advanced configuration: Model service warm-up