ApsaraVideo Live:Send and receive SEI messages

Overview

In video encoding, supplemental enhancement information (SEI) is part of video encoding standards, such as H.264/AVC and H.265/HEVC, and is used to transmit additional information related to the video content. This information enables auxiliary functions on the receiving end, such as content synchronization and error recovery. Common use cases include precise video layouts, synchronized lyrics, and live interactive quizzes.

SEI is the recommended method for sending custom messages. This method embeds the message into the video stream and transmits it along with the video data to other users in the channel.

• Benefits:

  • Real-time synchronization: Messages are synchronized with the video stream.

  • Unlimited receivers: Any user who subscribes to the stream can receive the messages.

• Limitations:

  • To avoid impacting the video stream, each message has a maximum size of 4 KB.

Sample code

ARTC provides an open-source sample project for your reference. You can download the project or view the code in the following repositories:

Android: Android/ARTCExample/BasicUsage/src/main/java/com/aliyun/artc/api/basicusage/SEIUsage/SEIActivity.java.

iOS: iOS/ARTCExample/BasicUsage/SEIUsage/SEIUsageVC.swift.

Harmony: Harmony/ARTCExample/entry/src/main/ets/pages/basicusage/SEIPage.ets.

Before you begin

Before you implement this feature, ensure you meet the following requirements:

• You have an active Alibaba Cloud account and have created an ARTC application. For instructions, see Create an ARTC application. You will need the AppID and AppKey, which are available in the ApsaraVideo Live console.

• You have integrated the ARTC SDK into your project and implemented basic audio and video calls. For more information, see Integrate the ARTC SDK and implement basic audio and video call features.

Implementation

// You must have already joined a channel and started publishing a video stream (which is enabled by default).
mSendSEIButton.setOnClickListener(v -> {
    if(mAliRtcEngine == null) {
        return;
    }
    String seiMessage = mEditText.getText().toString();
    if (TextUtils.isEmpty(seiMessage)) {
        return;
    }
    byte[] seiData = seiMessage.getBytes();
    mAliRtcEngine.sendMediaExtensionMsg(seiData,1,0,false);
    // mAliRtcEngine.sendMediaExtensionMsgEx(seiData,1,0,false, 5);
});

// Send an SEI message.
func sendSEI(seiMessage: String) -> Bool {
    guard let data = seiMessage.data(using: .utf8) else {
        return false
    }
    let repeatCount: Int32 = 1
    let delay: Int32 = 0
    let payloadType: Int32 = 5
    let isKeyFrameOnly = false
    let ret = self.rtcEngine?.sendMediaExtensionMsg(data, repeatCount: repeatCount, delay: delay, isKeyFrame: isKeyFrameOnly)
    // let ret = self.rtcEngine?.sendMediaExtensionMsgEx(data, repeatCount: repeatCount, delay: delay, isKeyFrame: isKeyFrameOnly, payloadType: payloadType)
    debugPrint("sendSEI: \(ret ?? -1)")
    return ret == 0
}

const seiData = this.stringToArrayBuffer(this.SendText);
// Send an SEI message.
this.rtcEngine.sendMediaExtensionMsg(seiData, 1, 0, false);

NSDateFormatter *formatter = [[NSDateFormatter alloc] init];
[formatter setDateFormat:@"YYYY-MM-dd HH:mm:ss"];
NSDate *datenow = [NSDate date];
NSString *currentTimeString = [formatter stringFromDate:datenow];
NSString *str =@"SEI Test time:" ;

NSString * ExtensionMsg = [str stringByAppendingString:currentTimeString];

NSData *data =[ExtensionMsg dataUsingEncoding:NSUTF8StringEncoding];

int code = [self.engine sendMediaExtensionMsg:data repeatCount:repectCount delay:100 isKeyFrame:true];
NSLog(@"--%d",code);

/* Specify the following parameters as needed. */
char * data = "xxxxx";
int length = strlen(data);
mAliRtcEngine->SendMediaExtensionMsg(data, length, 1,0,true);

@Override
public void onMediaExtensionMsgReceived(String uid, int payloadType, byte[]message) {
    super.onMediaExtensionMsgReceived(uid,payloadType, message);

    handler.post(new Runnable() {
        @Override
        public void run() {
            // Process the message.
            String receivedMsg = new String(message);
            ToastHelper.showToast(SEIActivity.this, receivedMsg, Toast.LENGTH_SHORT);
            
        }
    });
}

extension SEIUsageMainVC: AliRtcEngineDelegate {
    
    // Callback for receiving an SEI message.
    func onMediaExtensionMsgReceived(_ uid: String, payloadType: Int32, message data: Data) {
        // Process the message.
        guard let message = String(data: data, encoding: .utf8) else {
            print("Failed to parse the message")
            return
        }
        self.showToast(message: "Received SEI: \(message), from uid: \(uid), payloadType: \(payloadType)")
    }
    // Other callbacks...

}

// Callback for receiving an SEI message.
listener.onMediaExtensionMsgReceived((uid: string, payloadType: number, message: ArrayBuffer) => {
  console.info(`Received SEI message: uid=${uid}, payloadType=${payloadType}, message=${message}`);
});

- (void)onMediaExtensionMsgReceived:(NSString *)uid payloadType:(int)payloadType message:(NSData *)data {
    
    NSString * string_message  = [[NSString alloc] initWithBytes:data.bytes length:data.length encoding:NSUTF8StringEncoding];

    NSLog(@"received uid:%@, payloadType: %d, message:%@ size:%lu \n\n", uid, payloadType, string_message, (unsigned long)data.length);
    dispatch_async(dispatch_get_main_queue(), ^{
        NSString *showString = [NSString stringWithFormat:@" uid:%@\n payloadtype:%d message:%@\n size:%lu\n",
                                uid, payloadType, string_message, (unsigned long)data.length];
        _extendinfoL.stringValue = showString;
    });
    
}

public:
virtual void OnMediaExtensionMsgReceived(const char* uid, const int8_t * message, uint32_t size) override {
    /* TODO: Add your processing logic here based on your business requirements. */
}