Send and receive SEI messages

Overview

In video encoding, supplemental enhancement information (SEI) is part of the H.264/AVC and H.265/HEVC standards. SEI carries additional data alongside video content to enable functions such as content synchronization and error recovery. Common use cases include precise video layouts, synchronized lyrics, and live interactive quizzes.

SEI is the recommended method for sending custom messages. It embeds messages into the video stream for delivery to other users in the channel.

• Benefits:

  • Real-time synchronization: Messages are synchronized with the video stream.

  • Unlimited receivers: Any user who subscribes to the stream can receive the messages.

• Limitations:

  • Maximum message size is 4 KB to avoid impacting the video stream.

Sample code

ARTC provides open-source sample projects. Download or view the code in the following repositories:

Android: Android/ARTCExample/BasicUsage/src/main/java/com/aliyun/artc/api/basicusage/SEIUsage/SEIActivity.java.

iOS: iOS/ARTCExample/BasicUsage/SEIUsage/SEIUsageVC.swift.

Harmony: Harmony/ARTCExample/entry/src/main/ets/pages/basicusage/SEIPage.ets.

Before you begin

Ensure that you meet the following requirements:

• You have an Alibaba Cloud account and have created an ARTC application (Create an ARTC application). Obtain the AppID and AppKey from the ApsaraVideo Live console.

• You have integrated the ARTC SDK (Integrate the ARTC SDK) and implemented basic audio and video calls (implement basic audio and video call features).

Implementation

// You must have already joined a channel and started publishing a video stream (which is enabled by default).
mSendSEIButton.setOnClickListener(v -> {
    if(mAliRtcEngine == null) {
        return;
    }
    String seiMessage = mEditText.getText().toString();
    if (TextUtils.isEmpty(seiMessage)) {
        return;
    }
    byte[] seiData = seiMessage.getBytes();
    mAliRtcEngine.sendMediaExtensionMsg(seiData,1,0,false);
    // mAliRtcEngine.sendMediaExtensionMsgEx(seiData,1,0,false, 5);
});

// Send an SEI message.
func sendSEI(seiMessage: String) -> Bool {
    guard let data = seiMessage.data(using: .utf8) else {
        return false
    }
    let repeatCount: Int32 = 1
    let delay: Int32 = 0
    let payloadType: Int32 = 5
    let isKeyFrameOnly = false
    let ret = self.rtcEngine?.sendMediaExtensionMsg(data, repeatCount: repeatCount, delay: delay, isKeyFrame: isKeyFrameOnly)
    // let ret = self.rtcEngine?.sendMediaExtensionMsgEx(data, repeatCount: repeatCount, delay: delay, isKeyFrame: isKeyFrameOnly, payloadType: payloadType)
    debugPrint("sendSEI: \(ret ?? -1)")
    return ret == 0
}

const seiData = this.stringToArrayBuffer(this.SendText);
// Send an SEI message.
this.rtcEngine.sendMediaExtensionMsg(seiData, 1, 0, false);

NSDateFormatter *formatter = [[NSDateFormatter alloc] init];
[formatter setDateFormat:@"YYYY-MM-dd HH:mm:ss"];
NSDate *datenow = [NSDate date];
NSString *currentTimeString = [formatter stringFromDate:datenow];
NSString *str =@"SEI Test time:" ;

NSString * ExtensionMsg = [str stringByAppendingString:currentTimeString];

NSData *data =[ExtensionMsg dataUsingEncoding:NSUTF8StringEncoding];

int code = [self.engine sendMediaExtensionMsg:data repeatCount:repectCount delay:100 isKeyFrame:true];
NSLog(@"--%d",code);

/* Specify the following parameters as needed. */
char * data = "xxxxx";
int length = strlen(data);
mAliRtcEngine->SendMediaExtensionMsg(data, length, 1,0,true);

@Override
public void onMediaExtensionMsgReceived(String uid, int payloadType, byte[]message) {
    super.onMediaExtensionMsgReceived(uid,payloadType, message);

    handler.post(new Runnable() {
        @Override
        public void run() {
            // Process the message.
            String receivedMsg = new String(message);
            ToastHelper.showToast(SEIActivity.this, receivedMsg, Toast.LENGTH_SHORT);
            
        }
    });
}

extension SEIUsageMainVC: AliRtcEngineDelegate {
    
    // Callback for receiving an SEI message.
    func onMediaExtensionMsgReceived(_ uid: String, payloadType: Int32, message data: Data) {
        // Process the message.
        guard let message = String(data: data, encoding: .utf8) else {
            print("Failed to parse the message")
            return
        }
        self.showToast(message: "Received SEI: \(message), from uid: \(uid), payloadType: \(payloadType)")
    }
    // Other callbacks...

}

// Callback for receiving an SEI message.
listener.onMediaExtensionMsgReceived((uid: string, payloadType: number, message: ArrayBuffer) => {
  console.info(`Received SEI message: uid=${uid}, payloadType=${payloadType}, message=${message}`);
});

- (void)onMediaExtensionMsgReceived:(NSString *)uid payloadType:(int)payloadType message:(NSData *)data {
    
    NSString * string_message  = [[NSString alloc] initWithBytes:data.bytes length:data.length encoding:NSUTF8StringEncoding];

    NSLog(@"received uid:%@, payloadType: %d, message:%@ size:%lu \n\n", uid, payloadType, string_message, (unsigned long)data.length);
    dispatch_async(dispatch_get_main_queue(), ^{
        NSString *showString = [NSString stringWithFormat:@" uid:%@\n payloadtype:%d message:%@\n size:%lu\n",
                                uid, payloadType, string_message, (unsigned long)data.length];
        _extendinfoL.stringValue = showString;
    });
    
}

public:
virtual void OnMediaExtensionMsgReceived(const char* uid, const int8_t * message, uint32_t size) override {
    /* TODO: Add your processing logic here based on your business requirements. */
}