This topic explains how to integrate the Native RTS SDK with a third-party, FFmpeg-based player on iOS to implementRTS. This guide uses ijkplayer (tag k0.8.8) as an example.
Prerequisites
You must have compiled the ijkplayer source code. For instructions, see the README.md file in the ijkplayer repository.
Procedure
-
Download and decompress the ijkplayer source code. For the download link, see ijkplayer.
-
Download and decompress the Native RTS SDK. For the download link, see Release notes.
-
Integrate the Native RTS SDK into ijkplayer as a plug-in. There are two integration methods:
Integration method
Description
Advantage
Disadvantage
Extend FFmpeg.
Extend the FFmpeg demuxer in ijkplayer.
Easier to use. No special logic is required to handle ARTC URLs.
Requires recompiling the FFmpeg library.
Extend ijkplayer.
Add an AVInputFormat to ijkplayer.
Does not require compiling FFmpeg.
Requires adding custom logic to
ff_ffplay.c. -
Compile the IJKMediaPlayer project.
After completing the steps for either the FFmpeg extension or ijkplayer extension method, build the
ijkplayer/ios/IJKMediaPlayer/IJKMediaPlayer.xcodeprojproject using the Archive action. This action generates theIJKMediaFramework.frameworkfile in the Products directory.Optional: To support streams over HTTPS, you must import the previously generated
libssl.aandlibcrypto.afiles from the ijkplayer/ios/build/universal/lib directory. In the project's Build Phases, add them toLink Binary With Librariesbefore you compile. -
Import the RtsSDK.framework from the Native RTS SDK and the IJKMediaFramework.framework from ijkplayer into your custom project.
In your custom Xcode project, drag the
RtsSDK.frameworkandIJKMediaFramework.frameworkfiles into the project directory and add them to your target's dependency libraries. -
Call ijkplayer APIs to implement theRTS feature.
-
Create the ijkplayer instance
// Generate an ARTC protocol URL _url = @"artc://xxxx"; // Create a custom playerView [self.view addSubview:self.playerView]; ... IJKFFOptions *options = [IJKFFOptions optionsByDefault]; // Hardware decoding [options setPlayerOptionIntValue:1 forKey:@"videotoolbox"]; // Software decoding //[options setPlayerOptionIntValue:0 forKey:@"videotoolbox"]; _ijkPlayer = [[IJKFFMoviePlayerController alloc] initWithContentURL:[NSURL URLWithString:_url] withOptions:options]; _ijkPlayer.view.autoresizingMask = UIViewAutoresizingFlexibleWidth|UIViewAutoresizingFlexibleHeight; _ijkPlayer.view.frame = self.playerView.bounds; _ijkPlayer.scalingMode = IJKMPMovieScalingModeAspectFit; // Scaling mode _ijkPlayer.shouldAutoplay = YES; // Enable autoplay [self.playerView addSubview:_ijkPlayer.view]; -
Control playback
-
Start playback.
Playback must be started on the main thread. Before restarting playback, explicitly stop and release the player.
dispatch_async(dispatch_get_main_queue(), ^{ [_ijkPlayer prepareToPlay]; [_ijkPlayer play]; }); -
Stop playback and release the player.
// Stop playback [_ijkPlayer stop]; // Release the player [_ijkPlayer shutdown]; _ijkPlayer = nil;
-
-
Listen for RTS events
-
Listen for RTS message callbacks
Open the
ijkplayer/ios/IJKMediaPlayer/IJKMediaPlayer.xcodeprojproject and add the necessary adaptation methods to the relevant files.-
Modify the
ff_ffplay.cfile by inserting the following code block after thestatic int audio_open(FFPlayer *opaque, int64_t wanted_channel_layout, int wanted_nb_channels, int wanted_sample_rate, struct AudioParams *audio_hw_params){}method.extern int artcDemuxerMessage(struct AVFormatContext *s, int type, void *data, size_t data_size); // Aliyun RTS: Receive ARTC message int onArtcDemuxerMessage(struct AVFormatContext *s, int type, void *data, size_t data_size) { return artcDemuxerMessage(s, type, data, data_size); } int artcDemuxerMessage(struct AVFormatContext *s, int type, void *data, size_t data_size) { // Aliyun RTS: Send message to the app FFPlayer *ffp = (FFPlayer *)s->opaque; const char *data_msg = (const char *)data; ffp_notify_msg4(ffp,FFP_MSG_ARTC_DIRECTCOMPONENTMSG,type,0,data_msg,data_size); return 0; } -
Modify the
ff_ffplay.cfile by inserting the following RTS code block into thestatic int is_realtime(AVFormatContext *s){}method.static int is_realtime(AVFormatContext *s) { if( !strcmp(s->iformat->name, "rtp") || !strcmp(s->iformat->name, "rtsp") || !strcmp(s->iformat->name, "sdp") // *** RTS code block begin *** || !strcmp(s->iformat->name, "artc") // *** RTS code block end *** ) return 1; if(s->pb && ( !strncmp(s->filename, "rtp:", 4) || !strncmp(s->filename, "udp:", 4) ) ) return 1; return 0; } -
Modify the
ff_ffplay.cfile by inserting the following RTS code blocks into thestatic int read_thread(void *arg){}method.static int read_thread(void *arg) { ...... ic = avformat_alloc_context(); if (!ic) { av_log(NULL, AV_LOG_FATAL, "Could not allocate context.\n"); ret = AVERROR(ENOMEM); goto fail; } // *** RTS code block begin *** ic->opaque = ffp; ic->control_message_cb = onArtcDemuxerMessage; // *** RTS code block end *** ...... if (ffp->skip_calc_frame_rate) { av_dict_set_int(&ic->metadata, "skip-calc-frame-rate", ffp->skip_calc_frame_rate, 0); av_dict_set_int(&ffp->format_opts, "skip-calc-frame-rate", ffp->skip_calc_frame_rate, 0); } // *** RTS code block begin *** if(strncmp(is->filename, "artc://", 7) == 0) { extern AVInputFormat ff_rtc_demuxer; is->iformat = &ff_rtc_demuxer; } else { if(ffp->iformat_name) is->iformat = av_find_input_format(ffp->iformat_name); } // *** RTS code block end *** ...... pkt->flags = 0; // *** RTS code block begin *** if(strncmp(is->filename, "artc://", 7) == 0) { bool videoExist = is->video_stream >= 0; bool audioExist = is->audio_stream >= 0; // av_log(NULL, AV_LOG_INFO, "videoDuration %lld audioDuration %lld rate %f videoframeQue %d audioFrameque %d\n", // is->videoq.duration, is->audioq.duration, ffp->pf_playback_rate, // frame_queue_nb_remaining(&is->pictq), frame_queue_nb_remaining(&is->sampq)); if(!videoExist) { if(is->audioq.duration > 300 ) { // accelerate if(ffp->pf_playback_rate <= 1.0) { ffp->pf_playback_rate = 1.3; ffp->pf_playback_rate_changed = 1; av_log(NULL, AV_LOG_INFO, "aliyun rts set rate to %f\n", ffp->pf_playback_rate); } } else if(is->audioq.duration < 200) { // restore speed if(ffp->pf_playback_rate > 1.0) { ffp->pf_playback_rate = 1.0; ffp->pf_playback_rate_changed = 1; av_log(NULL, AV_LOG_INFO, "aliyun rts restore rate 1.0\n"); } } } else if((!videoExist || (videoExist && is->videoq.duration > 300)) && (!audioExist || (audioExist && is->audioq.duration > 300))) { if(ffp->pf_playback_rate <= 1) { ffp->pf_playback_rate = 1.3; ffp->pf_playback_rate_changed = 1; av_log(NULL, AV_LOG_INFO, "aliyun rts set rate 1.1\n"); } } else if((videoExist && is->videoq.duration <= 100) || (audioExist && is->audioq.duration <= 100)){ if(ffp->pf_playback_rate > 1) { ffp->pf_playback_rate = 1; ffp->pf_playback_rate_changed = 1; av_log(NULL, AV_LOG_INFO, "aliyun rts set rate 1\n"); } } } // *** RTS code block end *** ...... } -
Modify the
ff_ffmsg.hfile by adding the interface declaration for RTS messages.#define FFP_MSG_ARTC_DIRECTCOMPONENTMSG 3000 -
Send RTS messages to the upper layer.
-
Modify the
IJKMediaPlayback.hfile to add the name declaration for the RTS message listener.IJK_EXTERN NSString *const IJKMPMoviePlayerRtsMsgNotification; -
Modify the
IJKMediaPlayback.mfile to add the name definition for the RTS message listener.NSString *const IJKMPMoviePlayerRtsMsgNotification = @"IJKMPMoviePlayerRtsMsgNotification"; -
Modify the
IJKFFMoviePlayerController.mfile to add handling for the RTS message listener within thepostEvent:method.- (void)postEvent: (IJKFFMoviePlayerMessage *)msg { ...... case FFP_MSG_ARTC_DIRECTCOMPONENTMSG:{ NSString *rtsMsg = [[NSString alloc] initWithUTF8String:avmsg->obj]; int type = avmsg->arg1; if (!rtsMsg) { rtsMsg = @""; } NSDictionary *dic = @{@"type":@(type),@"msg":rtsMsg}; [[NSNotificationCenter defaultCenter] postNotificationName:IJKMPMoviePlayerRtsMsgNotification object:dic]; break; } default: // NSLog(@"unknown FFP_MSG_xxx(%d)\n", avmsg->what); break; } -
Build the
IJKMediaPlayer.xcodeprojproject using the Archive action to generate the latestIJKMediaFramework.framework.
-
-
-
Add an RTS retry method
-
Modify the
ff_ffplay.hfile to add the definition of therts_reload_flagvariable.int rts_reload_flag; -
Modify the
ff_ffplay.cfile by inserting the following code block after thestatic int audio_open(FFPlayer *opaque, int64_t wanted_channel_layout, int wanted_nb_channels, int wanted_sample_rate, struct AudioParams *audio_hw_params){}method.extern int artc_reload(AVFormatContext *ctx); void ffp_rts_reload(FFPlayer *ffp){ if(rts_reload_flag == 0) { rts_reload_flag = 1; } } -
Modify the
ff_ffplay.cfile by inserting the following RTS code block into thestatic int read_thread(void *arg){}method.static int read_thread(void *arg) { ...... #ifdef FFP_MERGE if (is->paused != is->last_paused) { is->last_paused = is->paused; if (is->paused) is->read_pause_return = av_read_pause(ic); else av_read_play(ic); } #endif // *** RTS code block begin *** if(rts_reload_flag){ rts_reload_flag = 0; av_log(ffp, AV_LOG_ERROR, "param == ffp_rts_reload\n"); VideoState *is = ffp->is; AVFormatContext *ic = is->ic; artc_reload(ic); } // *** RTS code block end *** ...... } -
Modify the
ijkplayer.hfile to add the definition of theijkmp_rts_reloadmethod.void ijkmp_rts_reload(IjkMediaPlayer *mp); -
Modify the
ijkplayer.cfile to implement theijkmp_rts_reloadmethod.void ijkmp_rts_reload(IjkMediaPlayer *mp) { ffp_rts_reload(mp->ffplayer); } -
Add the
rtsReloadinterface to the upper layer.-
Modify the
IJKFFMoviePlayerController.hfile to add the declaration of thertsReloadmethod.- (void)rtsReload; -
Modify the
IJKFFMoviePlayerController.mfile to implement thertsReloadmethod.- (void)rtsReload { ijkmp_rts_reload(_mediaPlayer); } -
Build the
IJKMediaPlayer.xcodeprojproject using the Archive action to generate the latestIJKMediaFramework.framework.
-
-
-
Implement stream degradation
-
Listen for the RTS message callback.
[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(reviceMsg:) name:IJKMPMoviePlayerRtsMsgNotification object:nil]; -
Implement the stream degradation logic in the RTS message callback method.
Stream degradation is a strategy that changes the player's URL prefix from
artc://tortmp://or to thehttp://xxxx.flvformat. The player then updates its source URL to resume playback.// Stream degradation to a traditional live streaming URL, such as http://xxx.flv or rtmp://xxx - (void)convertArtcToRtmpPlay { // Get the current playback URL and extract its prefix NSArray *urlSeparated = [self.url componentsSeparatedByString:@"://"]; NSString *urlPrefix = urlSeparated.firstObject; // Check if the URL prefix is "artc". If so, degrade to a traditional live stream if ([urlPrefix isEqualToString:@"artc"]) { // http://xxx.flv is recommended _url = [[@"http://" stringByAppendingString:urlSeparated.lastObject] stringByAppendingString:@".flv"]; // rtmp://xxx // _url = [@"rtmp://" stringByAppendingString:urlSeparated.lastObject]; // Stop playback and destroy the player [_ijkPlayer stop]; [_ijkPlayer shutdown]; _ijkPlayer = nil; // Reset the playback source _ijkPlayer = [[IJKFFMoviePlayerController alloc] initWithContentURL:[NSURL URLWithString:_url] withOptions:options]; ...... // Start playback dispatch_async(dispatch_get_main_queue(), ^{ [_ijkPlayer prepareToPlay]; [_ijkPlayer play]; }); } }You must first import the RtsSDK API.
#import <RtsSDK/rts_messages.h>Then, handle the player event callback.
-(void)reviceMsg:(NSNotification*)notification{ NSDictionary *dic = [notification object]; NSNumber *type = dic[@"type"]; switch (type.intValue) { case E_DNS_FAIL: case E_AUTH_FAIL: case E_CONN_TIMEOUT: case E_SUB_TIMEOUT: case E_SUB_NO_STREAM: { // Degrade the stream [self convertArtcToRtmpPlay]; } break; case E_STREAM_BROKEN: { static BOOL retryStartPlay = YES; // On the first RTS media timeout, retry playback once. On a subsequent timeout, degrade the stream directly. if (retryStartPlay) { dispatch_async(dispatch_get_main_queue(), ^{ [_ijkPlayer rtsReload]; }); retryStartPlay = NO; } else { // Degrade the stream [self convertArtcToRtmpPlay]; } } break; case E_RECV_STOP_SIGNAL: { // Stop playback and destroy the player [_ijkPlayer stop]; [_ijkPlayer shutdown]; _ijkPlayer = nil; } break; default: break; } }
-