ffplay源码剖析(3.2.4 + sdl2)：解码

Posted 2023-02-15 Ven_J

上一篇讲到ffplay的初始化和解复用，在解复用过程中创建了三个解码线程：音频、视频和字幕解码线程。同时将解复用的数据包分别添加到了音频包队列audioq、视频包队列videoq和字幕包队列subtitleq中。今天要讲的是从这三个包队列中分别进行解码，然后将解码出来的数据帧添加到对应的帧队列。显示线程就是从帧队列中取出数据帧然后进行显示。

read_thead函数中用于解码的函数是stream_component_open。

stream_component_open就是打开一个数据流，然后初始化解码器，然后创建真正的解码线程进行解码。

    /* open the streams */
    if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) 						//找到音频流索引，打开音频流
        stream_component_open(is, st_index[AVMEDIA_TYPE_AUDIO]);
    

    ret = -1;
    if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) 						//找到视频流索引，打开视频流
        ret = stream_component_open(is, st_index[AVMEDIA_TYPE_VIDEO]);
    
    if (is->show_mode == SHOW_MODE_NONE)
        is->show_mode = ret >= 0 ? SHOW_MODE_VIDEO : SHOW_MODE_RDFT;

    if (st_index[AVMEDIA_TYPE_SUBTITLE] >= 0) 						//找到字幕流索引，打开字幕流
        stream_component_open(is, st_index[AVMEDIA_TYPE_SUBTITLE]);
    

    if (is->video_stream < 0 && is->audio_stream < 0) 				//无法找到视频流和音频流
        av_log(NULL, AV_LOG_FATAL, "Failed to open file '%s' or configure filtergraph\\n",
               is->filename);
        ret = -1;
        goto fail;
    

上面的代码就是分别打开3个数据流进行解码。来看看stream_component_open函数的源码

/* open a given stream. Return 0 if OK */
/* 打开一个数据流，然后创建线程进行解码 */
static int stream_component_open(VideoState *is, int stream_index)

    AVFormatContext *ic = is->ic;
    AVCodecContext *avctx;
    AVCodec *codec;
    const char *forced_codec_name = NULL;
    AVDictionary *opts = NULL;
    AVDictionaryEntry *t = NULL;
    int sample_rate, nb_channels;
    int64_t channel_layout;
    int ret = 0;
    int stream_lowres = lowres;

    if (stream_index < 0 || stream_index >= ic->nb_streams)
        return -1;

    avctx = avcodec_alloc_context3(NULL);				//创建解码上下文
    if (!avctx)
        return AVERROR(ENOMEM);

    ret = avcodec_parameters_to_context(avctx, ic->streams[stream_index]->codecpar);	//将数据流的参数传递到解码上下文中
    if (ret < 0)
        goto fail;
    av_codec_set_pkt_timebase(avctx, ic->streams[stream_index]->time_base);		

    codec = avcodec_find_decoder(avctx->codec_id);			//根据解码上下文的解码id寻找解码器

    switch(avctx->codec_type)								//看命令行是否有指定解码器
        case AVMEDIA_TYPE_AUDIO   : is->last_audio_stream    = stream_index; forced_codec_name =    audio_codec_name; break;
        case AVMEDIA_TYPE_SUBTITLE: is->last_subtitle_stream = stream_index; forced_codec_name = subtitle_codec_name; break;
        case AVMEDIA_TYPE_VIDEO   : is->last_video_stream    = stream_index; forced_codec_name =    video_codec_name; break;
    	
    if (forced_codec_name)	//指定解码器
        codec = avcodec_find_decoder_by_name(forced_codec_name);	//根据指定解码器的名字寻找解码器
    if (!codec) 
        if (forced_codec_name) av_log(NULL, AV_LOG_WARNING,
                                      "No codec could be found with name '%s'\\n", forced_codec_name);
        else                   av_log(NULL, AV_LOG_WARNING,
                                      "No codec could be found with id %d\\n", avctx->codec_id);
        ret = AVERROR(EINVAL);
        goto fail;
    

    avctx->codec_id = codec->id;
    if(stream_lowres > av_codec_get_max_lowres(codec))
        av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\\n",
                av_codec_get_max_lowres(codec));
        stream_lowres = av_codec_get_max_lowres(codec);
    
    av_codec_set_lowres(avctx, stream_lowres);

    ......

    opts = filter_codec_opts(codec_opts, avctx->codec_id, ic, ic->streams[stream_index], codec);
    if (!av_dict_get(opts, "threads", NULL, 0))
        av_dict_set(&opts, "threads", "auto", 0);
    if (stream_lowres)
        av_dict_set_int(&opts, "lowres", stream_lowres, 0);
    if (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO)
        av_dict_set(&opts, "refcounted_frames", "1", 0);
    if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) 		//初始化解码器上下文
        goto fail;
    
    if ((t = av_dict_get(opts, "", NULL, AV_DICT_IGNORE_SUFFIX))) 
        av_log(NULL, AV_LOG_ERROR, "Option %s not found.\\n", t->key);
        ret =  AVERROR_OPTION_NOT_FOUND;
        goto fail;
    

    is->eof = 0;
    ic->streams[stream_index]->discard = AVDISCARD_DEFAULT;
    switch (avctx->codec_type) 			//根据解码器类型执行响应的解码
    case AVMEDIA_TYPE_AUDIO:

        sample_rate    = avctx->sample_rate;
        nb_channels    = avctx->channels;
        channel_layout = avctx->channel_layout;

        /* prepare audio output */
        if ((ret = audio_open(is, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0)
            goto fail;
        is->audio_hw_buf_size = ret;
        is->audio_src = is->audio_tgt;
        is->audio_buf_size  = 0;
        is->audio_buf_index = 0;

        /* init averaging filter */
        is->audio_diff_avg_coef  = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
        is->audio_diff_avg_count = 0;
        /* since we do not have a precise anough audio FIFO fullness,
           we correct audio sync only if larger than this threshold */
        is->audio_diff_threshold = (double)(is->audio_hw_buf_size) / is->audio_tgt.bytes_per_sec;

        is->audio_stream = stream_index;
        is->audio_st = ic->streams[stream_index];

        decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);	//初始化音频解码器
        if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) 
            is->auddec.start_pts = is->audio_st->start_time;
            is->auddec.start_pts_tb = is->audio_st->time_base;
        
        if ((ret = decoder_start(&is->auddec, audio_thread, is)) < 0)		//创建音频解码线程
            goto out;
        SDL_PauseAudio(0);
        break;
    case AVMEDIA_TYPE_VIDEO:
        is->video_stream = stream_index;
        is->video_st = ic->streams[stream_index];

        decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);	//初始化视频解码器
        if ((ret = decoder_start(&is->viddec, video_thread, is)) < 0)				//创建视频解码线程
            goto out;
        is->queue_attachments_req = 1;
        break;
    case AVMEDIA_TYPE_SUBTITLE:
        is->subtitle_stream = stream_index;
        is->subtitle_st = ic->streams[stream_index];

        decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread);	//初始化字幕解码器
        if ((ret = decoder_start(&is->subdec, subtitle_thread, is)) < 0)			//创建字幕解码线程
            goto out;
        break;
    default:
        break;
    
    goto out;

fail:
    avcodec_free_context(&avctx);
out:
    av_dict_free(&opts);

    return ret;

可以看出解码过程中会先调用decoder_init函数，然后调用decoder_start函数。decoder_init函数就是初始化解码器，将之前read_thread函数中的数据包队列和解码器联系起来，也就是解码器是从数据包队列中读取数据然后进行解码。

/* decoder_init - 解码器初始化 */
static void decoder_init(Decoder *d, AVCodecContext *avctx, PacketQueue *queue, SDL_cond *empty_queue_cond) 
    memset(d, 0, sizeof(Decoder));
    d->avctx = avctx;			//解码上下文
    d->queue = queue;			//数据包队列
    d->empty_queue_cond = empty_queue_cond;
    d->start_pts = AV_NOPTS_VALUE;

decoder_start就是开启解码过程，先启动包队列，创建解码线程。

/*启动解码过程*/
static int decoder_start(Decoder *d, int (*fn)(void *), void *arg)

    packet_queue_start(d->queue);		//开启包队列
    d->decoder_tid = SDL_CreateThread(fn, "decoder", arg);		//创建解码线程
    if (!d->decoder_tid) 
        av_log(NULL, AV_LOG_ERROR, "SDL_CreateThread(): %s\\n", SDL_GetError());
        return AVERROR(ENOMEM);
    
    return 0;

因为解复用和解码过程中都会用到包队列，解复用将数据包加入到队列中，解码将数据从队列中取出，这就是一个生产者和消费者的问题，因此会使用互斥锁来解决这里的同步问题。

创建的三个线程分别为audio_thread，video_thread，subtitle_thread。

其中audio_thread和subtitle_thread都是调用decoder_decode_frame进行解码得到数据帧，然后调用frame_queue_push将解码后的数据帧加入到帧队列中。

video_thread中调用get_video_frame函数获取视频帧，然后调用queue_picture将视频帧加入到视频帧队列中。实际上get_video_frame也是调用了decoder_decode_frame函数，因为视频帧不仅需要图像信息，还需要和音频进行同步，也就是pts的设置问题，get_video_frame中加入了对pts的更新操作。同理queue_picture也是调用了frame_queue_push函数，也加入了更新pts的操作。
到这一步，一共会同时运行有read_thread的解复用线程，audio_thread音频解码线程，video_thread视频解码线程，subtitle_thread字幕解码线程。

接下来还有一个显示线程以及音视频同步的问题会在之后的文章中更新。

因本人水平有限，如有不对之处还请批评指正。