ApsaraVideo Live:Java

Step 1: Import the SDK

The SDK package has the following directory structure:

AliRTCSDK_Linux/
└── Java/
    ├── libs/
    │   ├── AliRtcCoreService                      # Background service process (must specify an absolute path)
    │   ├── alirtc_linux_java_multiprocess.jar      # JAR bridge between the Java layer and the C++ engine
    │   ├── gson-2.11.0.jar                        # JSON serialization dependency
    │   └── libAliRtcLinuxEngine.so                # Complete dynamic library for the SDK
    └── Demo/
        ├── MainTest.java                          # Feature demo sample
        └── run.sh                                 # Script to compile and run

Compile and run the sample:

cd Demo
javac -g -cp ../libs/alirtc_linux_java_multiprocess.jar:../libs/gson-2.11.0.jar \
      -encoding utf-8 MainTest.java
java -cp .:../libs/alirtc_linux_java_multiprocess.jar:../libs/gson-2.11.0.jar MainTest

Before running, make sure the dynamic library can be found:

export LD_LIBRARY_PATH=/path/to/Java/libs:$LD_LIBRARY_PATH

Step 2: Implement the event callback class

Implement the AliRTCLinuxEngineListener interface to receive notifications from the SDK.

import com.alivc.rtc.multiprocess.AliRTCLinuxEngine;
import com.alivc.rtc.multiprocess.AliRTCLinuxEngineListener;

class VideoCallEventHandler implements AliRTCLinuxEngineListener {

    @Override
    public void onJoinChannelResult(int result, String channel, String userId) {
        if (result == 0) {
            System.out.printf("[onJoinChannelResult] User %s joined channel %s successfully%n",
                    userId, channel);
        } else {
            System.out.printf("[onJoinChannelResult] Failed to join, error: %d%n", result);
        }
    }

    @Override
    public void onRemoteUserOnLineNotify(String uid) {
        System.out.printf("[onRemoteUserOnLineNotify] uid: %s%n", uid);
    }

    @Override
    public void onRemoteUserOffLineNotify(String uid) {
        System.out.printf("[onRemoteUserOffLineNotify] uid: %s%n", uid);
    }

    @Override
    public void onRemoteTrackAvailableNotify(String uid,
            AliRTCLinuxEngine.AudioTrack audioTrack,
            AliRTCLinuxEngine.VideoTrack videoTrack) {
        System.out.printf("[onRemoteTrackAvailableNotify] uid: %s, audio: %s, video: %s%n",
                uid, audioTrack, videoTrack);
    }

    // onSubscribeMixedAudioFrame: Receives remote PCM audio frames after mixing.
    // Triggered when subscribeAudioFormat is set to AudioFormatMixedPcm in the subscription configuration.
    @Override
    public void onSubscribeMixedAudioFrame(AliRTCLinuxEngine.AudioFrame frame) {
        if (frame != null && frame.type == AliRTCLinuxEngine.AudioFrameType.AudioFrameRawPcm) {
            AliRTCLinuxEngine.AudioPcmFrame pcmFrame = frame.pcm;

            // pcmFrame.data        PCM data (byte[], int16_t format)

            // pcmFrame.channels    The number of sound channels
            // pcmFrame.sampleRates Sample rate
            // Write to a file, send to an audio device, or decode for playback here.
        }
    }

    // onSubscribeAudioFrame: Receives unmixed PCM frames from each remote user. The uid distinguishes the audio streams of different remote users.
    // Triggered when subscribeAudioFormat is set to AudioFormatPcmBeforeMixing in the subscription configuration.
    @Override
    public void onSubscribeAudioFrame(String uid, AliRTCLinuxEngine.AudioFrame frame) {
        if (frame != null && frame.type == AliRTCLinuxEngine.AudioFrameType.AudioFrameRawPcm) {
            AliRTCLinuxEngine.AudioPcmFrame pcmFrame = frame.pcm;
            // The uid identifies which remote user the frame is from.
            // Process audio data for each user separately here.
        }
    }

    // onRemoteVideoSample: Receives remote video frames.
    // The uid distinguishes the video streams of different remote users.
    @Override
    public void onRemoteVideoSample(String uid, AliRTCLinuxEngine.VideoFrame frame) {
        if (frame != null && frame.type == AliRTCLinuxEngine.VideoFrameType.VideoFrameH264) {
            AliRTCLinuxEngine.VideoH264Frame h264Frame = frame.h264;
            // The uid identifies which remote user the frame is from.
            // Write to a file, send to a renderer, or send to a video decoder here.
        }
    }

    @Override
    public void onError(int errorCode) {
        System.out.printf("[onError] error_code: 0x%X%n", errorCode);
    }
}

Step 3: Authentication token

The Java SDK has a built-in generateToken method. You can use it to directly generate a single-parameter token for joining a channel on the client.

Token generation process:

1. Concatenate the string: appId + appKey + channelId + userId + nonce + timestamp

2. Use SHA-256 hashing to generate a hexadecimal string.

3. Assemble the JSON: {"appid":..., "channelid":..., "userid":..., "nonce":..., "timestamp":..., "token":<sha256>}

4. Encode with Base64.

import java.time.Instant;

AliRTCLinuxEngine.AuthInfo authInfo = new AliRTCLinuxEngine.AuthInfo();
authInfo.appid    = "your_app_id";
authInfo.channel  = "your_channel_id";
authInfo.userid   = "your_user_id";
authInfo.username = "your_user_id";
authInfo.nonce    = null;
authInfo.timestamp = Instant.now().getEpochSecond() + 24 * 60 * 60; // Expires after 24 hours

String appKey = "your_app_key"; // In a production environment, do not expose the AppKey in the client code.

// Call generateToken to generate a single-parameter token for joining a channel (an engine instance must be created first).
authInfo.token = engineIns.generateToken(authInfo, appKey);

Step 4: Create and initialize the audio and video engine

You can call the AliRTCLinuxEngine.createInstance method to create an engine instance and pass in the event callback object.

VideoCallEventHandler listener = new VideoCallEventHandler();

String coreServicePath = "/path/to/Java/libs/AliRtcCoreService"; // Specify the absolute path of AliRtcCoreService
boolean h5mode = false; // Set to true to interoperate with web clients

String extra = "{\"user_specified_disable_audio_ranking\":\"true\"}";

AliRTCLinuxEngine engineIns = AliRTCLinuxEngine.createInstance(
    listener,
    42000, 45000,   // IPC port range for communication between the Java layer and the AliRtcCoreService process
    "/tmp",         // Log file directory
    coreServicePath,
    h5mode,
    extra
);

if (engineIns == null) {
    System.err.println("Failed to create RTC engine");
    return;
}

Step 5: Set audio and video properties

You can call the setClientRole method to set the user role. You can also call the setVideoEncoderConfiguration method to configure video encoding parameters.

// Call setClientRole to set the user role to interactive mode (streamer) to publish and subscribe at the same time.
engineIns.setClientRole(AliRTCLinuxEngine.AliEngineClientRole.AliEngineClientRoleInteractive);

// Call setVideoEncoderConfiguration to set video encoding parameters.
AliRTCLinuxEngine.AliEngineVideoEncoderConfiguration videoConfig =
        new AliRTCLinuxEngine.AliEngineVideoEncoderConfiguration();
videoConfig.dimensions = new AliRTCLinuxEngine.AliEngineVideoDimensions(720, 1280);
videoConfig.frameRate  = AliRTCLinuxEngine.AliEngineFrameRate.AliEngineFrameRateFps15;
videoConfig.bitrate    = 1200;
engineIns.setVideoEncoderConfiguration(videoConfig);

Step 6: Set stream ingest and pulling properties

Configure the publishing and subscription behavior for audio and video. Enable the external audio and video source mode, which is specific to the Linux platform.

// Call publishLocalVideoStream / publishLocalAudioStream to enable local audio and video publishing.
engineIns.publishLocalVideoStream(true);
engineIns.publishLocalAudioStream(true);

// Linux has no built-in camera or microphone. Call setExternalVideoSource to enable an external video source.
// Input YUV frame data using pushExternalVideoFrame.
engineIns.setExternalVideoSource(true,
        AliRTCLinuxEngine.VideoSource.VideoSourceCamera,
        AliRTCLinuxEngine.RenderMode.RenderModeFill);

// Call setExternalAudioSource to enable an external audio source. Input PCM frame data using pushExternalAudioFrameRawData.
engineIns.setExternalAudioSource(true, 16000 /* Sample rate */, 1 /* The number of sound channels */);

Configure the subscription mode for joining the channel. Set this in JoinChannelConfig in Step 7:

AliRTCLinuxEngine.JoinChannelConfig joinConfig = new AliRTCLinuxEngine.JoinChannelConfig();
joinConfig.channelProfile    = AliRTCLinuxEngine.ChannelProfile.ChannelProfileInteractiveLive;
joinConfig.publishMode       = AliRTCLinuxEngine.PublishMode.PublishAutomatically;     // Automatically publish streams
joinConfig.subscribeMode     = AliRTCLinuxEngine.SubscribeMode.SubscribeAutomatically; // Automatically subscribe to streams
joinConfig.publishAvsyncMode = AliRTCLinuxEngine.PublishAvsyncMode.PublishAvsyncWithPts;

// There are two options for the audio subscription format:
// AudioFormatMixedPcm: Receives mixed PCM data for the entire channel. This triggers onSubscribeMixedAudioFrame.
// AudioFormatPcmBeforeMixing: Receives individual PCM data from each remote user. This triggers onSubscribeAudioFrame (includes uid).
joinConfig.subscribeAudioFormat = AliRTCLinuxEngine.AudioFormat.AudioFormatMixedPcm;

// Receives H.264 video frames. This triggers onRemoteVideoSample.
joinConfig.subscribeVideoFormat = AliRTCLinuxEngine.VideoFormat.VideoFormatH264;

Step 7: Join the channel

You can call the joinChannel method and pass the token generated in Step 3 and the channel configuration to join the channel.

// Call joinChannel to join the channel.
engineIns.joinChannel(authInfo, joinConfig);

Step 8: Push external video frames

The Linux platform does not have a built-in camera driver interface. You must use an external input to send YUV video data to the SDK. The following example reads frame data in a loop from an I420 format YUV file and pushes the data. In a real-world business scenario, you can replace this with output from a camera driver or a video decoder.

Thread videoThread = new Thread(() -> {
    int width     = 720;
    int height    = 1280;
    int fps       = 15;
    int frameSize = width * height * 3 / 2; // I420

    try (FileInputStream fis = new FileInputStream("/tmp/test_720p.yuv")) {

        byte[ ] buf = new byte[frameSize];

        AliRTCLinuxEngine.VideoDataSample sample = new AliRTCLinuxEngine.VideoDataSample();

        while (running && !Thread.currentThread().isInterrupted()) {
            int readSize = fis.read(buf, 0, frameSize);
            if (readSize != frameSize) {
                fis.getChannel().position(0); // After the file is read, loop back to the beginning.
                continue;
            }

            sample.data       = buf;
            sample.format     = AliRTCLinuxEngine.VideoDataFormat.VideoDataFormatI420;
            sample.bufferType = AliRTCLinuxEngine.VideoBufferType.VideoBufferTypeRawData;
            sample.width      = width;
            sample.height     = height;
            sample.strideY    = width;
            sample.strideU    = width / 2;
            sample.strideV    = width / 2;
            sample.dataLen    = frameSize;
            sample.rotation   = 0;
            sample.timeStamp  = System.currentTimeMillis();

            int ret = engineIns.pushExternalVideoFrame(sample,
                    AliRTCLinuxEngine.VideoSource.VideoSourceCamera);
            if (ret < 0) break;

            Thread.sleep(1000 / fps);
        }
    } catch (IOException | InterruptedException e) {
        Thread.currentThread().interrupt();
    }
});
videoThread.start();

Step 9: Push external audio frames

The Linux platform does not have a built-in microphone recording interface. You must use an external input to send PCM audio data to the SDK. The following example reads frame data in a loop from a PCM file (16-bit integer, 16 kHz, mono) and pushes the data. In a real-world business scenario, you can replace this with output from a microphone driver or an audio decoder.

Thread audioThread = new Thread(() -> {
    int sampleRate = 16000;
    int channels   = 1;
    int frameMs    = 20;  // 20 ms per frame
    int frameSize  = (sampleRate / 1000) * frameMs * 2 * channels; // int16_t = 2 bytes

    try (FileInputStream fis = new FileInputStream("/tmp/test_16k_mono.pcm")) {

        byte[ ] buf = new byte[frameSize];


        while (running && !Thread.currentThread().isInterrupted()) {
            int readSize = fis.read(buf, 0, frameSize);
            if (readSize != frameSize) {
                fis.getChannel().position(0); // After the file is read, loop back to the beginning.
                continue;
            }

            int ret = engineIns.pushExternalAudioFrameRawData(buf, frameSize, 0);
            if (ret != 0) {
                // The SDK buffer is full. Move the file pointer back and retry later.
                fis.getChannel().position(fis.getChannel().position() - readSize);
                Thread.sleep(20);
                continue;
            }

            Thread.sleep(frameMs);
        }
    } catch (IOException | InterruptedException e) {
        Thread.currentThread().interrupt();
    }
});
audioThread.start();

Step 10: Handle remote audio and video playback

The Linux platform does not have built-in audio and video playback devices. Remote audio and video data is passed to the application layer through frame callbacks for custom processing. For example, you can write the data to a file, send it to a decoder, or connect it to a playback device.

Audio playback

Based on the subscribeAudioFormat configuration in Step 6, one of the following callbacks is triggered when a remote audio frame is received:

// AudioFormatMixedPcm mode: Receives mixed PCM data for the entire channel from all remote users.
@Override
public void onSubscribeMixedAudioFrame(AliRTCLinuxEngine.AudioFrame frame) {
    if (frame != null && frame.type == AliRTCLinuxEngine.AudioFrameType.AudioFrameRawPcm) {
        AliRTCLinuxEngine.AudioPcmFrame pcmFrame = frame.pcm;

        // pcmFrame.data        PCM data (byte[], int16_t format)

        // pcmFrame.channels    The number of sound channels
        // pcmFrame.sampleRates Sample rate
        // Write to a file, send to an audio device, or decode for playback here.
    }
}

// AudioFormatPcmBeforeMixing mode: Receives unmixed, individual PCM data from each user.
@Override
public void onSubscribeAudioFrame(String uid, AliRTCLinuxEngine.AudioFrame frame) {
    if (frame != null && frame.type == AliRTCLinuxEngine.AudioFrameType.AudioFrameRawPcm) {
        // The uid identifies which remote user the frame is from.
        // Process audio data for each user separately here.
    }
}

Video playback

When a remote video frame is received, the onRemoteVideoSample callback is triggered. The frame format is determined by the subscribeVideoFormat setting in Step 6:

@Override
public void onRemoteVideoSample(String uid, AliRTCLinuxEngine.VideoFrame frame) {
    if (frame != null && frame.type == AliRTCLinuxEngine.VideoFrameType.VideoFrameH264) {
        AliRTCLinuxEngine.VideoH264Frame h264Frame = frame.h264;
        // The uid identifies which remote user the frame is from.
        // Write to a file, send to a renderer, or send to a video decoder here.
    }
}

Step 11: Leave the channel and destroy the engine

To release resources correctly, you must stop stream ingest, leave the channel, and then destroy the engine, in that order.

// Stop the external stream ingest threads
running = false;
videoThread.interrupt();
audioThread.interrupt();
videoThread.join(1000);
audioThread.join(1000);

// Call publishLocalVideoStream(false) / publishLocalAudioStream(false) to stop publishing.
engineIns.publishLocalVideoStream(false);
engineIns.publishLocalAudioStream(false);

// Call leaveChannel to leave the channel.
engineIns.leaveChannel();

// Wait for the onLeaveChannelResult callback before destroying the engine.
// Call release to destroy the engine (must be called after leaveChannel).
engineIns.release();
engineIns = null;