Alibaba Cloud Model Studio:OpenAI compatible - Batch

Before you start a batch, use the test model batch-test-model to perform a full-link, closed-loop test. This test includes validating input data, creating a job, querying the job, and downloading the result file. Note:

• The test file must meet the requirements for an input file. It also must not exceed 1 MB in size or have more than 100 lines.

• Concurrency limit: A maximum of 2 parallel jobs.

• Resource usage: The test model does not go through the inference process and does not incur model inference fees.

The steps are as follows:

1. Prepare the test file

   • Download the sample file test_model.jsonl, which contains request information, to your local machine. Make sure that it is in the same directory as the Python script below.

   • Sample content: Set the model parameter to batch-test-model and the url parameter to /v1/chat/ds-test .

     {"custom_id":"1","method":"POST","url":"/v1/chat/ds-test","body":{"model":"batch-test-model","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello! How can I help you?"}]}}
     {"custom_id":"2","method":"POST","url":"/v1/chat/ds-test","body":{"model":"batch-test-model","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is 2+2?"}]}}

2. Run the script

   • Run this Python script.

     To adjust the file path or other parameters, modify the code as needed.

     import os
     from pathlib import Path
     from openai import OpenAI
     import time
     
     # Initialize the client
     client = OpenAI(
         # If you have not configured environment variables, replace the following line with api_key="sk-xxx" using your Model Studio API key. However, we do not recommend hard-coding the API key into your code in a production environment to reduce the risk of leaks.
         # The API keys for the Singapore and Beijing regions are different. To obtain an API key, see https://www.alibabacloud.com/help/en/model-studio/get-api-key
         api_key=os.getenv("DASHSCOPE_API_KEY"),
         # The following is the base_url for the Singapore region. If you use a model in the Beijing region, replace the base_url with https://dashscope.aliyuncs.com/compatible-mode/v1
         base_url="https://dashscope-intl.aliyuncs.com/compatible-mode/v1"  # The base_url for the Alibaba Cloud Model Studio service
     )
     
     def upload_file(file_path):
         print(f"Uploading the JSONL file that contains request information...")
         file_object = client.files.create(file=Path(file_path), purpose="batch")
         print(f"File uploaded successfully. File ID: {file_object.id}\n")
         return file_object.id
     
     def create_batch_job(input_file_id):
         print(f"Creating a batch job based on the file ID...")
         # Note: The value of the endpoint parameter here must be the same as the url field in the input file. For the test model (batch-test-model), enter /v1/chat/ds-test. For text embedding models, enter /v1/embeddings. For other models, enter /v1/chat/completions.
         batch = client.batches.create(input_file_id=input_file_id, endpoint="/v1/chat/ds-test", completion_window="24h")
         print(f"Batch job created successfully. Batch job ID: {batch.id}\n")
         return batch.id
     
     def check_job_status(batch_id):
         print(f"Checking the batch job status...")
         batch = client.batches.retrieve(batch_id=batch_id)
         print(f"Batch job status: {batch.status}\n")
         return batch.status
     
     def get_output_id(batch_id):
         print(f"Getting the output file ID for successful requests in the batch job...")
         batch = client.batches.retrieve(batch_id=batch_id)
         print(f"Output file ID: {batch.output_file_id}\n")
         return batch.output_file_id
     
     def get_error_id(batch_id):
         print(f"Getting the error file ID for failed requests in the batch job...")
         batch = client.batches.retrieve(batch_id=batch_id)
         print(f"Error file ID: {batch.error_file_id}\n")
         return batch.error_file_id
     
     def download_results(output_file_id, output_file_path):
         print(f"Printing and downloading the results of successful requests from the batch job...")
         content = client.files.content(output_file_id)
         # Print some content for testing
         print(f"Printing the first 1,000 characters of the successful results: {content.text[:1000]}...\n")
         # Save the result file to your local machine
         content.write_to_file(output_file_path)
         print(f"The complete output results have been saved to the local output file result.jsonl\n")
     
     def download_errors(error_file_id, error_file_path):
         print(f"Printing and downloading the information for failed requests from the batch job...")
         content = client.files.content(error_file_id)
         # Print some content for testing
         print(f"Printing the first 1,000 characters of the failed request information: {content.text[:1000]}...\n")
         # Save the error information file to your local machine
         content.write_to_file(error_file_path)
         print(f"The complete failed request information has been saved to the local error file error.jsonl\n")
     
     def main():
         # File paths
         input_file_path = "test_model.jsonl"  # Replace with your input file path
         output_file_path = "result.jsonl"  # Replace with your output file path
         error_file_path = "error.jsonl"  # Replace with your error file path
         try:
             # Step 1: Upload the JSONL file containing request information to get the input file ID
             input_file_id = upload_file(input_file_path)
             # Step 2: Create a batch job based on the input file ID
             batch_id = create_batch_job(input_file_id)
             # Step 3: Check the batch job status until it is finished
             status = ""
             while status not in ["completed", "failed", "expired", "cancelled"]:
                 status = check_job_status(batch_id)
                 print(f"Waiting for the job to complete...")
                 time.sleep(10)  # Wait 10 seconds before checking the status again
             # If the job fails, print the error message and exit
             if status == "failed":
                 batch = client.batches.retrieve(batch_id)
                 print(f"Batch job failed. Error information: {batch.errors}\n")
                 print(f"For more information, see the error codes documentation: https://www.alibabacloud.com/help/en/model-studio/developer-reference/error-code")
                 return
             # Step 4: Download the results. If an output file ID exists, print the first 1,000 characters of the successful results and download the complete results to a local output file.
             # If an error file ID exists, print the first 1,000 characters of the failed request information and download the complete information to a local error file.
             output_file_id = get_output_id(batch_id)
             if output_file_id:
                 download_results(output_file_id, output_file_path)
             error_file_id = get_error_id(batch_id)
             if error_file_id:
                 download_errors(error_file_id, error_file_path)
                 print(f"For more information, see the error codes documentation: https://www.alibabacloud.com/help/en/model-studio/developer-reference/error-code")
         except Exception as e:
             print(f"An error occurred: {e}")
             print(f"For more information, see the error codes documentation: https://www.alibabacloud.com/help/en/model-studio/developer-reference/error-code")
     
     if __name__ == "__main__":
         main()

3. Verify the test results

   • The job status shows completed.

   • Result file result.jsonl: Contains the fixed response {"content":"This is a test result."}.

     {"id":"a2b1ae25-21f4-4d9a-8634-99a29926486c","custom_id":"1","response":{"status_code":200,"request_id":"a2b1ae25-21f4-4d9a-8634-99a29926486c","body":{"created":1743562621,"usage":{"completion_tokens":6,"prompt_tokens":20,"total_tokens":26},"model":"batch-test-model","id":"chatcmpl-bca7295b-67c3-4b1f-8239-d78323bb669f","choices":[{"finish_reason":"stop","index":0,"message":{"content":"This is a test result."}}],"object":"chat.completion"}},"error":null}
     {"id":"39b74f09-a902-434f-b9ea-2aaaeebc59e0","custom_id":"2","response":{"status_code":200,"request_id":"39b74f09-a902-434f-b9ea-2aaaeebc59e0","body":{"created":1743562621,"usage":{"completion_tokens":6,"prompt_tokens":20,"total_tokens":26},"model":"batch-test-model","id":"chatcmpl-1e32a8ba-2b69-4dc4-be42-e2897eac9e84","choices":[{"finish_reason":"stop","index":0,"message":{"content":"This is a test result."}}],"object":"chat.completion"}},"error":null}

     If an error occurs, see Error messages to resolve it.

After you verify the test, follow these steps to run a formal batch job.

1. Prepare the input file according to the input file requirements. Set the model parameter in the file to a supported model and set the url parameter to: /v1/chat/completions

2. Replace the endpoint in the Python script above.

   Important

   Make sure that the endpoint in the script matches the url parameter in the input file.

3. Run the script and wait for the job to complete. If the job is successful, an output file named result.jsonl is generated in the same directory.

   If the job fails, the program exits and prints an error message.

   If an error file ID exists, an error file named error.jsonl is generated in the same directory for you to review.

   Exceptions that occur during the process are caught, and an error message is printed.

{"custom_id":"1","method":"POST","url":"/v1/chat/completions","body":{"model":"qwen-plus","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello!"}]}}
{"custom_id":"2","method":"POST","url":"/v1/chat/completions","body":{"model":"qwen-plus","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is 2+2?"}]}}

JSONL batch generation tool

Use this tool to quickly generate JSONL files. To avoid performance issues, do not process more than 10,000 rows at a time. If you have a large data volume, process the data in batches.

Input parameter settings

Response example

For more information, see the response example for Create a batch job.

Response parameters

For more information, see the response parameters for Create a batch job.

The content of output_file_id and error_file_id in the response parameters can be retrieved using Download the batch result file.

Input parameter settings

{
  "object": "list",
  "data": [
    {
      "id": "batch_xxx",
      "object": "batch",
      "endpoint": "/v1/chat/completions",
      "errors": null,
      "input_file_id": "file-batch-xxx",
      "completion_window": "24h",
      "status": "completed",
      "output_file_id": "file-batch_output-xxx",
      "error_file_id": null,
      "created_at": 1722234109,
      "in_progress_at": 1722234109,
      "expires_at": null,
      "finalizing_at": 1722234165,
      "completed_at": 1722234165,
      "failed_at": null,
      "expired_at": null,
      "cancelling_at": null,
      "cancelled_at": null,
      "request_counts": {
        "total": 100,
        "completed": 95,
        "failed": 5
      },
      "metadata": {}
    },
    { ... }
  ],
  "first_id": "batch_xxx",
  "last_id": "batch_xxx",
  "has_more": true
}

Input parameter settings

Response example

For more information, see the response example for Create a batch job.

Response parameters

For more information, see the response parameters for Create a batch job.