Alibaba Cloud Model Studio:Real-time speech synthesis - CosyVoice

import os
import time
import dashscope
from dashscope.audio.tts_v2 import VoiceEnrollmentService, SpeechSynthesizer

# 1. Prepare the environment.
# We recommend that you configure the API key using an environment variable.
# API keys for the Singapore and Beijing regions are different. To get an API key, see https://www.alibabacloud.com/help/en/model-studio/get-api-key
# If you have not configured environment variables, replace the following line with your Model Studio API key: dashscope.api_key = "sk-xxx"
dashscope.api_key = os.getenv("DASHSCOPE_API_KEY")
if not dashscope.api_key:
    raise ValueError("DASHSCOPE_API_KEY environment variable not set.")

# The following is the WebSocket URL for the Singapore region. If you use a model in the Beijing region, replace the URL with: wss://dashscope.aliyuncs.com/api-ws/v1/inference
dashscope.base_websocket_api_url='wss://dashscope-intl.aliyuncs.com/api-ws/v1/inference'
# The following is the HTTP URL for the Singapore region. If you use a model in the Beijing region, replace the URL with: https://dashscope.aliyuncs.com/api/v1
dashscope.base_http_api_url = 'https://dashscope-intl.aliyuncs.com/api/v1'


# 2. Define the cloning parameters.
TARGET_MODEL = "cosyvoice-v3.5-plus" 
# Give the voice a meaningful prefix.
VOICE_PREFIX = "myvoice" # Only digits and lowercase letters are allowed. The prefix must be less than 10 characters in length.
# A publicly accessible audio URL.
AUDIO_URL = "https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/cosyvoice/cosyvoice-zeroshot-sample.wav" # This is a sample URL. Replace it with your own.

# 3. Create a voice (asynchronous task).
print("--- Step 1: Creating voice enrollment ---")
service = VoiceEnrollmentService()
try:
    voice_id = service.create_voice(
        target_model=TARGET_MODEL,
        prefix=VOICE_PREFIX,
        url=AUDIO_URL
    )
    print(f"Voice enrollment submitted successfully. Request ID: {service.get_last_request_id()}")
    print(f"Generated Voice ID: {voice_id}")
except Exception as e:
    print(f"Error during voice creation: {e}")
    raise e
# 4. Poll for the voice status.
print("\n--- Step 2: Polling for voice status ---")
max_attempts = 30
poll_interval = 10 # seconds
for attempt in range(max_attempts):
    try:
        voice_info = service.query_voice(voice_id=voice_id)
        status = voice_info.get("status")
        print(f"Attempt {attempt + 1}/{max_attempts}: Voice status is '{status}'")
        
        if status == "OK":
            print("Voice is ready for synthesis.")
            break
        elif status == "UNDEPLOYED":
            print(f"Voice processing failed with status: {status}. Please check audio quality or contact support.")
            raise RuntimeError(f"Voice processing failed with status: {status}")
        # For intermediate statuses such as "DEPLOYING", continue to wait.
        time.sleep(poll_interval)
    except Exception as e:
        print(f"Error during status polling: {e}")
        time.sleep(poll_interval)
else:
    print("Polling timed out. The voice is not ready after several attempts.")
    raise RuntimeError("Polling timed out. The voice is not ready after several attempts.")

# 5. Use the cloned voice for speech synthesis.
print("\n--- Step 3: Synthesizing speech with the new voice ---")
try:
    synthesizer = SpeechSynthesizer(model=TARGET_MODEL, voice=voice_id)
    text_to_synthesize = "Congratulations, you have successfully cloned and synthesized your own voice!"
    
    # The call() method returns binary audio data.
    audio_data = synthesizer.call(text_to_synthesize)
    print(f"Speech synthesis successful. Request ID: {synthesizer.get_last_request_id()}")

    # 6. Save the audio file.
    output_file = "my_custom_voice_output.mp3"
    with open(output_file, "wb") as f:
        f.write(audio_data)
    print(f"Audio saved to {output_file}")

except Exception as e:
    print(f"Error during speech synthesis: {e}")

1. Generate a custom voice and preview the result. If you are satisfied with the result, proceed to the next step. Otherwise, regenerate the voice.

   Python

   import requests
   import base64
   import os
   
   def create_voice_and_play():
       # API keys for the Singapore and Beijing regions are different. To get an API key, see https://www.alibabacloud.com/help/en/model-studio/get-api-key
       # If you have not configured environment variables, replace the following line with your Model Studio API key: api_key = "sk-xxx"
       api_key = os.getenv("DASHSCOPE_API_KEY")
       
       if not api_key:
           print("Error: The DASHSCOPE_API_KEY environment variable is not found. Set the API key.")
           return None, None, None
       
       # Prepare the request data.
       headers = {
           "Authorization": f"Bearer {api_key}",
           "Content-Type": "application/json"
       }
       
       data = {
           "model": "voice-enrollment",
           "input": {
               "action": "create_voice",
               "target_model": "cosyvoice-v3.5-plus",
               "voice_prompt": "A composed middle-aged male announcer with a deep, rich and magnetic voice, a steady speaking speed and clear articulation, is suitable for news broadcasting or documentary commentary.",
               "preview_text": "Dear listeners, hello everyone. Welcome to the evening news.",
               "prefix": "announcer"
           },
           "parameters": {
               "sample_rate": 24000,
               "response_format": "wav"
           }
       }
       
       # The following is the URL for the Singapore region. If you use a model in the Beijing region, replace the URL with: https://dashscope.aliyuncs.com/api/v1/services/audio/tts/customization
       url = "https://dashscope-intl.aliyuncs.com/api/v1/services/audio/tts/customization"
       
       try:
           # Send the request.
           response = requests.post(
               url,
               headers=headers,
               json=data,
               timeout=60  # Add a timeout setting.
           )
           
           if response.status_code == 200:
               result = response.json()
               
               # Get the voice ID.
               voice_id = result["output"]["voice_id"]
               print(f"Voice ID: {voice_id}")
               
               # Get the preview audio data.
               base64_audio = result["output"]["preview_audio"]["data"]
               
               # Decode the Base64 audio data.
               audio_bytes = base64.b64decode(base64_audio)
               
               # Save the audio file to your on-premises device.
               filename = f"{voice_id}_preview.wav"
               
               # Write the audio data to a local file.
               with open(filename, 'wb') as f:
                   f.write(audio_bytes)
               
               print(f"The audio is saved to the local file: {filename}")
               print(f"File path: {os.path.abspath(filename)}")
               
               return voice_id, audio_bytes, filename
           else:
               print(f"Request failed. Status code: {response.status_code}")
               print(f"Response content: {response.text}")
               return None, None, None
               
       except requests.exceptions.RequestException as e:
           print(f"A network request error occurred: {e}")
           return None, None, None
       except KeyError as e:
           print(f"The response data is in an invalid format. The required field is missing: {e}")
           print(f"Response content: {response.text if 'response' in locals() else 'No response'}")
           return None, None, None
       except Exception as e:
           print(f"An unknown error occurred: {e}")
           return None, None, None
   
   if __name__ == "__main__":
       print("Creating the voice...")
       voice_id, audio_data, saved_filename = create_voice_and_play()
       
       if voice_id:
           print(f"\nVoice '{voice_id}' is created.")
           print(f"The audio file is saved: '{saved_filename}'")
           print(f"File size: {os.path.getsize(saved_filename)} bytes")
       else:
           print("\nFailed to create the voice.")

   Java

   You need to import the Gson dependency. If you use Maven or Gradle, add the dependency as follows:

   Maven

   Add the following to your pom.xml file:

   <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
   <dependency>
       <groupId>com.google.code.gson</groupId>
       <artifactId>gson</artifactId>
       <version>2.13.1</version>
   </dependency>

   Gradle

   Add the following to your build.gradle file:

   // https://mvnrepository.com/artifact/com.google.code.gson/gson
   implementation("com.google.code.gson:gson:2.13.1")

   import com.google.gson.JsonObject;
   import com.google.gson.JsonParser;
   import java.io.*;
   import java.net.HttpURLConnection;
   import java.net.URL;
   import java.util.Base64;
   
   public class Main {
       public static void main(String[] args) {
           Main example = new Main();
           example.createVoice();
       }
   
       public void createVoice() {
           // API keys for the Singapore and Beijing regions are different. To get an API key, see https://www.alibabacloud.com/help/en/model-studio/get-api-key
           // If you have not configured environment variables, replace the following line with your Model Studio API key: String apiKey = "sk-xxx"
           String apiKey = System.getenv("DASHSCOPE_API_KEY");
   
           // Create a JSON request body string.
           String jsonBody = "{\n" +
                   "    \"model\": \"voice-enrollment\",\n" +
                   "    \"input\": {\n" +
                   "        \"action\": \"create_voice\",\n" +
                   "        \"target_model\": \"cosyvoice-v3.5-plus\",\n" +
                   "        \"voice_prompt\": \"A composed middle-aged male announcer with a deep, rich and magnetic voice, a steady speaking speed and clear articulation, is suitable for news broadcasting or documentary commentary.\",\n" +
                   "        \"preview_text\": \"Dear listeners, hello everyone. Welcome to the evening news.\",\n" +
                   "        \"prefix\": \"announcer\"\n" +
                   "    },\n" +
                   "    \"parameters\": {\n" +
                   "        \"sample_rate\": 24000,\n" +
                   "        \"response_format\": \"wav\"\n" +
                   "    }\n" +
                   "}";
   
           HttpURLConnection connection = null;
           try {
               // The following is the URL for the Singapore region. If you use a model in the Beijing region, replace the URL with: https://dashscope.aliyuncs.com/api/v1/services/audio/tts/customization
               URL url = new URL("https://dashscope-intl.aliyuncs.com/api/v1/services/audio/tts/customization");
               connection = (HttpURLConnection) url.openConnection();
   
               // Set the request method and headers.
               connection.setRequestMethod("POST");
               connection.setRequestProperty("Authorization", "Bearer " + apiKey);
               connection.setRequestProperty("Content-Type", "application/json");
               connection.setDoOutput(true);
               connection.setDoInput(true);
   
               // Send the request body.
               try (OutputStream os = connection.getOutputStream()) {
                   byte[] input = jsonBody.getBytes("UTF-8");
                   os.write(input, 0, input.length);
                   os.flush();
               }
   
               // Get the response.
               int responseCode = connection.getResponseCode();
               if (responseCode == HttpURLConnection.HTTP_OK) {
                   // Read the response content.
                   StringBuilder response = new StringBuilder();
                   try (BufferedReader br = new BufferedReader(
                           new InputStreamReader(connection.getInputStream(), "UTF-8"))) {
                       String responseLine;
                       while ((responseLine = br.readLine()) != null) {
                           response.append(responseLine.trim());
                       }
                   }
   
                   // Parse the JSON response.
                   JsonObject jsonResponse = JsonParser.parseString(response.toString()).getAsJsonObject();
                   JsonObject outputObj = jsonResponse.getAsJsonObject("output");
                   JsonObject previewAudioObj = outputObj.getAsJsonObject("preview_audio");
   
                   // Get the voice name.
                   String voiceId = outputObj.get("voice_id").getAsString();
                   System.out.println("Voice ID: " + voiceId);
   
                   // Get the Base64-encoded audio data.
                   String base64Audio = previewAudioObj.get("data").getAsString();
   
                   // Decode the Base64 audio data.
                   byte[] audioBytes = Base64.getDecoder().decode(base64Audio);
   
                   // Save the audio to a local file.
                   String filename = voiceId + "_preview.wav";
                   saveAudioToFile(audioBytes, filename);
   
                   System.out.println("The audio is saved to the local file: " + filename);
   
               } else {
                   // Read the error response.
                   StringBuilder errorResponse = new StringBuilder();
                   try (BufferedReader br = new BufferedReader(
                           new InputStreamReader(connection.getErrorStream(), "UTF-8"))) {
                       String responseLine;
                       while ((responseLine = br.readLine()) != null) {
                           errorResponse.append(responseLine.trim());
                       }
                   }
   
                   System.out.println("Request failed. Status code: " + responseCode);
                   System.out.println("Error response: " + errorResponse.toString());
               }
   
           } catch (Exception e) {
               System.err.println("A request error occurred: " + e.getMessage());
               e.printStackTrace();
           } finally {
               if (connection != null) {
                   connection.disconnect();
               }
           }
       }
   
       private void saveAudioToFile(byte[] audioBytes, String filename) {
           try {
               File file = new File(filename);
               try (FileOutputStream fos = new FileOutputStream(file)) {
                   fos.write(audioBytes);
               }
               System.out.println("The audio is saved to: " + file.getAbsolutePath());
           } catch (IOException e) {
               System.err.println("An error occurred while saving the audio file: " + e.getMessage());
               e.printStackTrace();
           }
       }
   }

2. Use the custom voice you generated in the previous step for speech synthesis.

   This step references the non-streaming call example code. Replace the voice parameter with the custom voice generated through voice design for speech synthesis.

   Key principle: The model used for voice design (target_model) must match the model used for subsequent speech synthesis (model), or synthesis will fail.

   Python

   # coding=utf-8
   
   import dashscope
   from dashscope.audio.tts_v2 import *
   import os
   
   # API keys for the Singapore and Beijing regions are different. To get an API key, see https://www.alibabacloud.com/help/en/model-studio/get-api-key
   # If you have not configured environment variables, replace the following line with your Model Studio API key: dashscope.api_key = "sk-xxx"
   dashscope.api_key = os.environ.get('DASHSCOPE_API_KEY')
   
   # The following is the URL for the Singapore region. If you use a model in the Beijing region, replace the URL with: wss://dashscope.aliyuncs.com/api-ws/v1/inference
   dashscope.base_websocket_api_url='wss://dashscope-intl.aliyuncs.com/api-ws/v1/inference'
   
   # Use the same model for voice design and speech synthesis.
   model = "cosyvoice-v3.5-plus"
   # Replace the voice parameter with the custom voice generated by voice design.
   voice = "your_voice"
   
   # Instantiate SpeechSynthesizer and pass request parameters such as model and voice in the constructor.
   synthesizer = SpeechSynthesizer(model=model, voice=voice)
   # Send the text to be synthesized and get the binary audio.
   audio = synthesizer.call("How is the weather today?")
   # The first time you send text, a WebSocket connection is established. The first packet delay includes the connection establishment time.
   print('[Metric] Request ID: {}, First packet delay: {} ms'.format(
       synthesizer.get_last_request_id(),
       synthesizer.get_first_package_delay()))
   
   # Save the audio locally.
   with open('output.mp3', 'wb') as f:
       f.write(audio)

   Java

   import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam;
   import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer;
   import com.alibaba.dashscope.utils.Constants;
   
   import java.io.File;
   import java.io.FileOutputStream;
   import java.io.IOException;
   import java.nio.ByteBuffer;
   
   public class Main {
       // Use the same model for voice design and speech synthesis.
       private static String model = "cosyvoice-v3.5-plus";
       // Replace the voice parameter with the custom voice generated by voice design.
       private static String voice = "your_voice_id";
   
       public static void streamAudioDataToSpeaker() {
           // Request parameters
           SpeechSynthesisParam param =
                   SpeechSynthesisParam.builder()
                           // API keys for the Singapore and Beijing regions are different. To get an API key, see https://www.alibabacloud.com/help/en/model-studio/get-api-key
                           // If you have not configured environment variables, replace the following line with your Model Studio API key: .apiKey("sk-xxx")
                           .apiKey(System.getenv("DASHSCOPE_API_KEY"))
                           .model(model) // Model
                           .voice(voice) // Voice
                           .build();
   
           // Synchronous mode: Disable callback (second parameter is null).
           SpeechSynthesizer synthesizer = new SpeechSynthesizer(param, null);
           ByteBuffer audio = null;
           try {
               // Block until audio returns.
               audio = synthesizer.call("How is the weather today?");
           } catch (Exception e) {
               throw new RuntimeException(e);
           } finally {
               // Close the WebSocket connection when the task ends.
               synthesizer.getDuplexApi().close(1000, "bye");
           }
           if (audio != null) {
               // Save the audio data to the local file "output.mp3".
               File file = new File("output.mp3");
               // The first time you send text, a WebSocket connection is established. The first packet delay includes the connection establishment time.
               System.out.println(
                       "[Metric] Request ID: "
                               + synthesizer.getLastRequestId()
                               + ", First packet delay (ms): "
                               + synthesizer.getFirstPackageDelay());
               try (FileOutputStream fos = new FileOutputStream(file)) {
                   fos.write(audio.array());
               } catch (IOException e) {
                   throw new RuntimeException(e);
               }
           }
       }
   
       public static void main(String[] args) {
           // The following is the URL for the Singapore region. If you use a model in the Beijing region, replace the URL with: wss://dashscope.aliyuncs.com/api-ws/v1/inference
           Constants.baseWebsocketApiUrl = "wss://dashscope-intl.aliyuncs.com/api-ws/v1/inference";
           streamAudioDataToSpeaker();
           System.exit(0);
       }
   }