视频中音频包严重滞后的优化方案

Posted 涂程

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了视频中音频包严重滞后的优化方案相关的知识,希望对你有一定的参考价值。

好文推荐
作者:码上就说

播放流程介绍

播放视频的流程想必大家都非常清楚了,大家先看一下下面的视频播放的简单流程图:

解封装之后得到的数据包,有音频数据包和视频数据包,这时候一般会准备两个队列,一个存放视频数据包,一个存放音频数据包。

  • FFmpeg中使用av_read_frame来读取数据包,读出的数据包存放在 AVPacket中
  • 如果发现packet是音频,就放在音频队列中
  • 如果发现packet是视频,就放在视频队列中

队列的大小是有限制的?如果达到了队列大小的限制,就应该挂起当前的读取包的线程,等到解码线程解码队列中的数据包。

if (context->video_packet_queue->nb_packets >= context->video_packet_queue->max_size) 
  pthread_mutex_lock(&context->packet_full_mutex_t);
  pthread_cond_wait(&context->packet_full_cond_t, &context->packet_full_mutex_t);
  pthread_mutex_unlock(&context->packet_full_mutex_t);

因为视频数据比音频数据要大得多,所以锁住的也是视频队列,没有考虑到音频队列。

这时候会启动另外一个线程来取队列中的数据,这个线程的主要工作是负责解码。当队列中的数据包被取出来被解码之后,就要重新唤醒被锁住的线程,让它可以继续读取数据包。上面简单的介绍就是播放器的主要流程。

这时候数据队列的大小是一个值得考虑的事情了?

  • video packet queue
  • audio packet queue

一般视频的帧率是30fps,一秒视频30帧,视频队列设计成30 * 5 大小是比较合适的,可以存放5s的视频包数据。 这样的涉及在正常情况下是没有问题的,但是本文正好分析一种特殊的视频。

特殊视频介绍

正常的视频一般是音频packet和视频packet相间出现的,这样是正常的,用packet queue来控制音视频同步的时候不会出现问题。如下面出现的情况,不会出现只出现音频packet或者视频packet的情况。

packet,audio,1,-5058,-0.114694,-5058,-0.114694,2048,0.046440,186,103651,KD

packet,video,0,0,0.000000,-7200,-0.080000,3600,0.040000,6528,103837,K_

packet,audio,1,-3010,-0.068254,-3010,-0.068254,2048,0.046440,186,110365,KD

packet,video,0,7200,0.080000,-3600,-0.040000,3600,0.040000,11475,110551,__

packet,audio,1,-962,-0.021814,-962,-0.021814,2048,0.046440,186,122026,K_

packet,video,0,3600,0.040000,0,0.000000,3600,0.040000,124,122212,__

packet,audio,1,1086,0.024626,1086,0.024626,2048,0.046440,185,122336,K_

packet,video,0,14400,0.160000,3600,0.040000,3600,0.040000,19436,122521,__

packet,audio,1,3134,0.071066,3134,0.071066,2048,0.046440,186,141957,K_

packet,video,0,10800,0.120000,7200,0.080000,3600,0.040000,161,142143,__

packet,audio,1,5182,0.117506,5182,0.117506,2048,0.046440,272,142304,K_

packet,video,0,25200,0.280000,10800,0.120000,3600,0.040000,12728,142576,__

packet,video,0,18000,0.200000,14400,0.160000,3600,0.040000,163,155304,__

packet,audio,1,7230,0.163946,7230,0.163946,2048,0.046440,250,155467,K_

packet,video,0,21600,0.240000,18000,0.200000,3600,0.040000,189,155717,__

packet,audio,1,9278,0.210385,9278,0.210385,2048,0.046440,236,155906,K_

packet,video,0,39600,0.440000,21600,0.240000,3600,0.040000,4613,156142,__

packet,audio,1,11326,0.256825,11326,0.256825,2048,0.046440,231,160755,K_

packet,video,0,32400,0.360000,25200,0.280000,3600,0.040000,887,160986,__

packet,audio,1,13374,0.303265,13374,0.303265,2048,0.046440,212,161873,K_

packet,video,0,28800,0.320000,28800,0.320000,3600,0.040000,491,162085,__

packet,audio,1,15422,0.349705,15422,0.349705,2048,0.046440,220,162576,K_

packet,video,0,36000,0.400000,32400,0.360000,3600,0.040000,532,162796,__

packet,audio,1,17470,0.396145,17470,0.396145,2048,0.046440,237,163328,K_

但是还是会有一些非常特殊的视频,音频packet严重滞后,是封装的时候出现了问题,下面的例子就是前面基本上全是视频packet,没有出现音频packet,这样音频的packet queue会被撑爆的,针对这种情况,播放视频的时候要设计好队列模式和音视频同步机制,防止出现画面一直卡住的问题。

packet,video,0,2048,0.135593,768,0.050847,256,0.016949,2010,32204,__

packet,video,0,1536,0.101695,1024,0.067797,256,0.016949,286,34214,__

packet,audio,1,3072,0.069660,3072,0.069660,1024,0.023220,456,34500,K_

packet,video,0,1280,0.084746,1280,0.084746,256,0.016949,135,34956,__

packet,audio,1,4096,0.092880,4096,0.092880,1024,0.023220,465,35091,K_

packet,video,0,1792,0.118644,1536,0.101695,256,0.016949,147,35556,__

packet,audio,1,5120,0.116100,5120,0.116100,1024,0.023220,431,35703,K_

packet,video,0,3072,0.203390,1792,0.118644,256,0.016949,3726,36134,__

packet,video,0,2560,0.169492,2048,0.135593,256,0.016949,214,39860,__

packet,audio,1,6144,0.139320,6144,0.139320,1024,0.023220,404,40074,K_

packet,video,0,2304,0.152542,2304,0.152542,256,0.016949,858,40478,__

packet,video,0,2816,0.186441,2560,0.169492,256,0.016949,96,41336,__

packet,video,0,4096,0.271186,2816,0.186441,256,0.016949,3472,41432,__

packet,video,0,3584,0.237288,3072,0.203390,256,0.016949,607,44904,__

packet,video,0,3328,0.220339,3328,0.220339,256,0.016949,519,45511,__

packet,video,0,3840,0.254237,3584,0.237288,256,0.016949,104,46030,__

packet,video,0,5120,0.338983,3840,0.254237,256,0.016949,945,46134,__

packet,video,0,4608,0.305085,4096,0.271186,256,0.016949,99,47079,__

packet,video,0,4352,0.288136,4352,0.288136,256,0.016949,117,47178,__

packet,video,0,4864,0.322034,4608,0.305085,256,0.016949,46,47295,__

packet,video,0,6144,0.406780,4864,0.322034,256,0.016949,1462,47341,__

packet,video,0,5632,0.372881,5120,0.338983,256,0.016949,253,48803,__

packet,video,0,5376,0.355932,5376,0.355932,256,0.016949,321,49056,__

packet,video,0,5888,0.389831,5632,0.372881,256,0.016949,94,49377,__

packet,video,0,7168,0.474576,5888,0.389831,256,0.016949,809,49471,__

packet,video,0,6656,0.440678,6144,0.406780,256,0.016949,146,50280,__

packet,video,0,6400,0.423729,6400,0.423729,256,0.016949,56,50426,__

packet,video,0,6912,0.457627,6656,0.440678,256,0.016949,105,50482,__

packet,video,0,8192,0.542373,6912,0.457627,256,0.016949,1851,50587,__

packet,video,0,7680,0.508475,7168,0.474576,256,0.016949,145,52438,__

packet,video,0,7424,0.491525,7424,0.491525,256,0.016949,336,52583,__

packet,video,0,7936,0.525424,7680,0.508475,256,0.016949,48,52919,__

packet,video,0,9216,0.610169,7936,0.525424,256,0.016949,771,52967,__

packet,video,0,8704,0.576271,8192,0.542373,256,0.016949,118,53738,__

packet,video,0,8448,0.559322,8448,0.559322,256,0.016949,71,53856,__

packet,video,0,8960,0.593220,8704,0.576271,256,0.016949,46,53927,__

packet,video,0,9984,0.661017,8960,0.593220,256,0.016949,268,53973,__

packet,video,0,9472,0.627119,9216,0.610169,256,0.016949,71,54241,__

packet,video,0,9728,0.644068,9472,0.627119,256,0.016949,44,54312,__

packet,video,0,10240,0.677966,9728,0.644068,256,0.016949,35307,54356,__

packet,video,0,11264,0.745763,9984,0.661017,256,0.016949,3893,89663,__

packet,video,0,10752,0.711864,10240,0.677966,256,0.016949,440,93556,__

packet,video,0,10496,0.694915,10496,0.694915,256,0.016949,144,93996,__

packet,video,0,11008,0.728814,10752,0.711864,256,0.016949,359,94140,__

packet,audio,1,7168,0.162540,7168,0.162540,1024,0.023220,390,94499,K_

packet,video,0,12288,0.813559,11008,0.728814,256,0.016949,5313,94889,__

packet,video,0,11776,0.779661,11264,0.745763,256,0.016949,516,100202,__

packet,video,0,11520,0.762712,11520,0.762712,256,0.016949,207,100718,__

packet,video,0,12032,0.796610,11776,0.779661,256,0.016949,106,100925,__

packet,video,0,13312,0.881356,12032,0.796610,256,0.016949,4026,101031,__

packet,video,0以上是关于视频中音频包严重滞后的优化方案的主要内容,如果未能解决你的问题,请参考以下文章

vLine 媒体会话中的视频滞后于音频

从 URL 滞后片段加载 VideoView

Java 基于 JAVE 库 实现 视频转音频的批量转换

Exoplayer 音频播放背景,甚至视频已关闭

BottomNavigationView 滞后于片段事务

一次用ffmpeg实现图片+音频合成视频的开发