Alibaba Cloud Service Mesh:Deploy a large language model as an inference service

from typing import List

from mlserver import MLModel, types
from mlserver.codecs import decode_args

from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import os

class PeftModelServer(MLModel):
    async def load(self) -> bool:
        self._load_model()
        self.ready = True
        return self.ready

    @decode_args
    async def predict(self, content: List[str]) -> List[str]:
        return self._predict_outputs(content)

    def _load_model(self):
        model_name_or_path = os.environ.get("PRETRAINED_MODEL_PATH", "bigscience/bloomz-560m")
        peft_model_id = os.environ.get("PEFT_MODEL_ID", "aipipeline/bloomz-560m_PROMPT_TUNING_CAUSAL_LM")
        self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, local_files_only=True)
        config = PeftConfig.from_pretrained(peft_model_id)
        self.model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)
        self.model = PeftModel.from_pretrained(self.model, peft_model_id)
        self.text_column = os.environ.get("DATASET_TEXT_COLUMN_NAME", "Tweet text")
        return

    def _predict_outputs(self, content: List[str]) -> List[str]:
        output_list = []
        for input in content:
            inputs = self.tokenizer(
                f'{self.text_column} : {input} Label : ',
                return_tensors="pt",
            )
            with torch.no_grad():
                inputs = {k: v for k, v in inputs.items()}
                outputs = self.model.generate(
                    input_ids=inputs["input_ids"], attention_mask=inputs["attention_mask"], max_new_tokens=10, eos_token_id=3
                )
                outputs = self.tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)
            output_list.append(outputs[0])
        return output_list

# TODO: choose appropriate base image, install Python, MLServer, and
# dependencies of your MLModel implementation
FROM python:3.8-slim-buster
RUN pip install mlserver peft transformers datasets
# ...

# The custom MLModel implementation should be on the Python search path
# instead of relying on the working directory of the image. If using a
# single-file module, this can be accomplished with:
COPY --chown=${USER} ./peft_model_server.py /opt/peft_model_server.py
ENV PYTHONPATH=/opt/

# environment variables to be compatible with ModelMesh Serving
# these can also be set in the ServingRuntime, but this is recommended for
# consistency when building and testing
ENV MLSERVER_MODELS_DIR=/models/_mlserver_models \
    MLSERVER_GRPC_PORT=8001 \
    MLSERVER_HTTP_PORT=8002 \
    MLSERVER_LOAD_MODELS_AT_STARTUP=false \
    MLSERVER_MODEL_NAME=peft-model

# With this setting, the implementation field is not required in the model
# settings which eases integration by allowing the built-in adapter to generate
# a basic model settings file
ENV MLSERVER_MODEL_IMPLEMENTATION=peft_model_server.PeftModelServer

CMD mlserver start ${MLSERVER_MODELS_DIR}

apiVersion: serving.kserve.io/v1alpha1
kind: ServingRuntime
metadata:
  name: peft-model-server
  namespace: modelmesh-serving
spec:
  supportedModelFormats:
    - name: peft-model
      version: "1"
      autoSelect: true
  multiModel: true
  grpcDataEndpoint: port:8001
  grpcEndpoint: port:8085
  containers:
    - name: mlserver
      image:  registry.cn-beijing.aliyuncs.com/test/peft-model-server:latest
      env:
        - name: MLSERVER_MODELS_DIR
          value: "/models/_mlserver_models/"
        - name: MLSERVER_GRPC_PORT
          value: "8001"
        - name: MLSERVER_HTTP_PORT
          value: "8002"
        - name: MLSERVER_LOAD_MODELS_AT_STARTUP
          value: "true"
        - name: MLSERVER_MODEL_NAME
          value: peft-model
        - name: MLSERVER_HOST
          value: "127.0.0.1"
        - name: MLSERVER_GRPC_MAX_MESSAGE_LENGTH
          value: "-1"
        - name: PRETRAINED_MODEL_PATH
          value: "bigscience/bloomz-560m"
        - name: PEFT_MODEL_ID
          value: "aipipeline/bloomz-560m_PROMPT_TUNING_CAUSAL_LM"
        # - name: "TRANSFORMERS_OFFLINE"
        #   value: "1"
        # - name: "HF_DATASETS_OFFLINE"
        #   value: "1"
      resources:
        requests:
          cpu: 500m
          memory: 4Gi
        limits:
          cpu: "5"
          memory: 5Gi
  builtInAdapter:
    serverType: mlserver
    runtimeManagementPort: 8001
    memBufferBytes: 134217728
    modelLoadingTimeoutMillis: 90000