Make FFMPEG to work with HLS adaptive streams

Hello. Today I want to tell you how I implement adaptive stream into FFMPEG. Not so long ago I had to make android app which had to play hls streams. I used IJKPplayer . It’s wrapper of FFMPEG. I don’t want to talk a lot why I chose exactly this player, but the main reason was that only this player can play 4 video streams at the same time. 

 If you read this article, I think, you know what is HLS, but I will shortly tell about it. We receive file which contains streams links and a player has to change this streams depending to current bandwidth.

Example:

       

#EXTM3U
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=688301
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/0640_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=165135
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/0150_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=262346
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/0240_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=481677
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/0440_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=1308077
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/1240_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=1927853
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/1840_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=2650941
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/2540_vod.m3u8
#EXT-X-STREAM-INF:PROGRAM-ID=1, BANDWIDTH=3477293
http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/3340_vod.m3u8

 

After some work with IJK, we found that IJKPlayer doesn’t change streams. One ijkplayer’s developer suggested to parse stream outside of player and use the one you need(also the same ticket I found on ffmepg). Of Course it doesn’t work for me. In web I didn’t find something that was able to help me. So I had to fix this issue by myself. I looked inside of FFMPEG and allocated several points which we have to know.

• There’s method read_data, which locates in libavformat/hls.c, it's responsible for all magic. There player loads stream and push it into buffer. At the end of the method you can see goto restart, where happens changing of segment.  We will change the stream before this restart if we need.
• Second place which we are interested in, it’s libavformat/avio.c. It contains a method ffurl_close, which is called when stream have to be closed. There we will calculate current bandwidth. Also it contains method ffurl_open, which, of course, opens your stream, so there we will reset our counter of loaded bites and stop timer.

Ok, let’s recap our tasks:

• Calculate current bandwidth
• Change stream, depend of current bandwidth
• Clean data when user will stop watch a video

Listing:

bitrate_manager.h

       

#include 
#ifndef IJKPLAYER_TEST_H
#define IJKPLAYER_TEST_H

extern int64_t start_loading;
extern int64_t end_loading ;
extern int64_t loaded_bytes;
extern int64_t currentBitrate;
extern int64_t diff;

//array of links 
extern char** urls;
//array of bandwidth of links above
extern int64_t* bandwidth;
extern int n_arrays_items;
extern char* selected_url;
extern int current_url_index;
extern int64_t current_bandwidth;

void saveStartLoadingData();

int64_t getStartLoading();

//check that manager was initialized 
int isInited();

//add to counter count of loaded bytes for one time
void addToLoadingByte(int64_t bytesCount);

//end of loading current segment. Calculate time which was spent for loading current segment
void endOfLoading();

//calculate current bandwidth
void calculateAndSaveCurrentBitrate();

int64_t getDiff();

int64_t getLoadedBites();

int64_t getEndLoading();

int64_t getCurrentBitrate();

void setFullUrl(char* url);
void setParturlParts();

//Check do we have variants, or no
int doWeHaveBadwidth();
//create array with streams links 
void createDataArrays(int n_items);

//fill links array
void addData(int i, char* url, int64_t band_width);

//free memory
void freeData();

//return current selected link
char* getCurrentUrl();

//compare link with  current selected link
int compareUrl(char* url);

//find stream for current bandwidth
void findBestSolutionForCurrentBandwidth();

char* getUrlString(int index);

#endif //IJKPLAYER_TEST_H
 

bitrate_manager.c

       

#include "bitrate_manager.h"
#include 
#include 
#include 
#include "libavutil/log.h"

static const int64_t ONE_SECOND= 1000000000LL;

int64_t start_loading;
int64_t end_loading ;
int64_t loaded_bytes;
int64_t currentBitrate;
int64_t diff;

char** urls;
int64_t* bandwidth;
int n_arrays_items;
char* selected_url;
int current_url_index;
int64_t current_bandwidth;

/*
 * It contains current last index + 1
 */
int pointerAfterLastItem;

int isInitedData = 0;

int64_t now_ms() {
    struct timespec now;
    clock_gettime(CLOCK_MONOTONIC, &now);
    return (int64_t) now.tv_sec*1000000000LL + now.tv_nsec;
}

void saveStartLoadingData(){
    loaded_bytes = 0LL;
    start_loading =  now_ms();
}

int64_t getStartLoading(){
    return start_loading;
}

int isInited(){
    return isInitedData;
}

void addToLoadingByte(int64_t bytesCount){
    loaded_bytes += bytesCount;
}

void endOfLoading(){
    end_loading = now_ms();
    diff = end_loading - start_loading;
}

void calculateAndSaveCurrentBitrate(){
    if(loaded_bytes != 0) {
        currentBitrate = loaded_bytes * ONE_SECOND / diff;
    }
    loaded_bytes = 0;
}

int64_t getDiff(){
    return diff;
}
int64_t getLoadedBites(){
    return loaded_bytes;
}
int64_t getEndLoading(){
    return end_loading;
}
int64_t getCurrentBitrate(){
    return currentBitrate;
}

int doWeHaveBadwidth(){
    if(bandwidth && pointerAfterLastItem != 0){
        return 1;
    }
    return 0;
}
void createDataArrays(int n_items){
    isInitedData = 1;
    pointerAfterLastItem = 0;
    n_arrays_items = n_items;
    bandwidth = (int64_t*) malloc(n_items * sizeof(int64_t));
    urls = (char**) malloc(n_items * sizeof(char*));
    for(int i =0; i < n_items; i++){
        urls[i] = (char*) malloc(sizeof(char));
    }
}

void addData(int i, char* url, int64_t band_width){
    if(band_width == 0LL){
        return;
    }
    free(urls[i]);
    urls[i] = (char*) malloc(strlen(url) * sizeof(char));
    strcpy(urls[pointerAfterLastItem], url);
    bandwidth[pointerAfterLastItem] = band_width;
    pointerAfterLastItem++;
}

void freeData(){
    if(isInitedData == 0){
        return;
    }
    isInitedData = 0;
    for(int i = 0;i < pointerAfterLastItem;++i) free(urls[i]);
    free(urls);
    free(bandwidth);
}

char* getCurrentUrl(){
    return selected_url;
}

int compareUrl(char* url){
    if(selected_url){
        return strcmp(selected_url, url);
    }
    return 0;
}

void findBestSolutionForCurrentBandwidth() {
    if (currentBitrate == 0) {
        selected_url = urls[0];
        current_url_index = 0;
        current_bandwidth = bandwidth[0];
        return;
    }
    if (currentBitrate == current_bandwidth) return;

    int index = 0;
    int64_t selectedBitrate = bandwidth[index];
    int start = 0;
    int length = pointerAfterLastItem;
    for (int i = start; i < length; i++) {
       if (currentBitrate >= bandwidth[i]
            && selectedBitrate <= bandwidth[i]) {
            index = i;
            selectedBitrate = bandwidth[i];
        }
    }
    if (current_bandwidth != selectedBitrate) {
        selected_url = urls[index];
        current_url_index = index;
        current_bandwidth = selectedBitrate;
    }
}
 

Listing of ffmpeg
In avio.c we will add:

 avio.c

       

int ffurl_open(URLContext **puc, const char *filename, int flags,
               const AVIOInterruptCB *int_cb, AVDictionary **options)
{
    if(isInited() == 1) {
        saveStartLoadingData();
    }
 ….
}
….

int ffurl_close(URLContext *h)
{
    if( isInited() == 1) {
        endOfLoading();
        calculateAndSaveCurrentBitrate();
    }
    return ffurl_closep(&h);
}
 

In hls.c read_data will look like this

 hls.c

       

static int read_data(void *opaque, uint8_t *buf, int buf_size)
{
    struct playlist *v = opaque;
    HLSContext *c = v->parent->priv_data;

// init playlist
    if (isInited() == 0) {
        createDataArrays(c->n_variants);
        for (int i = 0; i < c->n_variants; i++) {
             addData(i, c->playlists[i]->url, c->variants[i]->bandwidth);
        }
    }
//change stream if we need
    if(doWeHaveBadwidth() == 1 && isInited() == 1 && compareUrl(v->url) != 0){
        strcpy(v->url, getCurrentUrl());
    }
    
    int ret, i;
    int just_opened = 0;

restart:
    if (!v->needed)
        return AVERROR_EOF;

    if (!v->input) {
        int64_t reload_interval;

        /* Check that the playlist is still needed before opening a new
         * segment. */
        if (v->ctx && v->ctx->nb_streams &&
            v->parent->nb_streams >= v->stream_offset + v->ctx->nb_streams) {
            v->needed = 0;
            for (i = v->stream_offset; i < v->stream_offset + v->ctx->nb_streams;
                i++) {
                if (v->parent->streams[i]->discard < AVDISCARD_ALL)
                    v->needed = 1;
            }
        }
        if (!v->needed) {
            av_log(v->parent, AV_LOG_INFO, "No longer receiving playlist %d\n",
                v->index);
            return AVERROR_EOF;
        }

        /* If this is a live stream and the reload interval has elapsed since
         * the last playlist reload, reload the playlists now. */
        reload_interval = default_reload_interval(v);

reload:
        if (!v->finished &&
            av_gettime_relative() - v->last_load_time >= reload_interval) {
            if ((ret = parse_playlist(c, v->url, v, NULL)) < 0) {
                av_log(v->parent, AV_LOG_WARNING, "Failed to reload playlist %d\n",
                       v->index);
                return ret;
            }
//add count of loaded bytes to counter
            if(isInited() == 1 && doWeHaveBadwidth() == 1) {
                addToLoadingByte(ret);
            }
            /* If we need to reload the playlist again below (if
             * there's still no more segments), switch to a reload
             * interval of half the target duration. */
            reload_interval = v->target_duration / 2;
        }
        if (v->cur_seq_no < v->start_seq_no
              || v->cur_seq_no > (v->start_seq_no + (v->n_segments * 5)) ) {
            av_log(NULL, AV_LOG_WARNING,
                   "skipping %d segments ahead, expired from playlists\n",
                   v->start_seq_no - v->cur_seq_no);
            v->cur_seq_no = v->start_seq_no;
        }
        if (v->cur_seq_no >= v->start_seq_no + v->n_segments) {
            if (v->finished)
                return AVERROR_EOF;
            while (av_gettime_relative() - v->last_load_time < reload_interval) {
                if (ff_check_interrupt(c->interrupt_callback))
                    return AVERROR_EXIT;
                av_usleep(100*1000);
            }
            /* Enough time has elapsed since the last reload */
            goto reload;
        }

        ret = open_input(c, v);
//add count of loaded bytes to counter
        if(isInited() == 1 && doWeHaveBadwidth() == 1) {
            addToLoadingByte(ret);
        }
        if (ret < 0) {
            if (ff_check_interrupt(c->interrupt_callback))
                return AVERROR_EXIT;
            av_log(v->parent, AV_LOG_WARNING, "Failed to open segment of playlist %d\n",
                   v->index);
            v->cur_seq_no += 1;
            goto reload;
        }
        just_opened = 1;
    }

    ret = read_from_url(v, buf, buf_size, READ_NORMAL);
//add count of loaded bytes to counter
if(isInited() == 1 && doWeHaveBadwidth() == 1) {
        addToLoadingByte(ret);
    }
    if (ret > 0) {
        if (just_opened && v->is_id3_timestamped != 0) {
            /* Intercept ID3 tags here, elementary audio streams are required
             * to convey timestamps using them in the beginning of each segment. */
            intercept_id3(v, buf, buf_size, &ret);
        }

        return ret;
    }
    ffurl_close(v->input);
    v->input = NULL;
    v->cur_seq_no++;

    c->cur_seq_no = v->cur_seq_no;
// data loading is finished. Looking for stream for current bandwidth and if it’s differ from current, switch to new stream
    if(isInited() == 1
           && doWeHaveBadwidth() == 1) {
        findBestSolutionForCurrentBandwidth();
        if (compareUrl(v->url) != 0) {
            strcpy(v->url, getCurrentUrl());
        }
    }
    goto restart;
}
 

Ok, only several things left. Add new files into makefile inside of libavformat.

makefile

       

NAME = avformat

HEADERS = avformat.h                                                    \
          avio.h                                                        \
          version.h                                                     \
          avc.h                                                         \
          url.h                                                         \
          internal.h                                                    \
          bitrate_mamnger.h                                                        \


OBJS = allformats.o         \
       avio.o               \
       aviobuf.o            \
       cutils.o             \
       dump.o               \
       format.o             \
       id3v1.o              \
       id3v2.o              \
       metadata.o           \
       mux.o                \
       options.o            \
       os_support.o         \
       riff.o               \
       sdp.o                \
       url.o                \
       utils.o              \
       avc.o                \
       bitrate_mamnger.o               \
                                 

At the end add a method to IjkMediaPlayer_freeBitateWorkData in ijkplayer_jni.c, which will be called when user will stop watching video to clean data.

 ijkplayer_jni.c

       

static void
IjkMediaPlayer_freeBitateWorkData(JNIEnv *env, jclass clazz){
    freeData();
}
//and add current method to g_methods
...
{ "_freeBitateWorkData", "()V",  (void *)IjkMediaPlayer_freeBitateWorkData },
...
 

That’s it, our implementation is done. The only thing left is to rebuild the project. Now you should be able to watch the video with adaptive streams.