Libav AVFrame 到 Opencv Mat 到 AVPacket 的转换
Posted
技术标签:
【中文标题】Libav AVFrame 到 Opencv Mat 到 AVPacket 的转换【英文标题】：Libav AVFrame to Opencv Mat to AVPacket conversion 【发布时间】：2017-06-30 17:35:58 【问题描述】：我是 libav 的新手，我正在编写一个以 opencv 为核心的视频处理软件。我所做的简要如下：
1-读取视频包
2-将数据包解码成AVFrame
3- 转换 AVFrame 到 CV Mat
4- 操纵垫子
5- 转换 CV Mat 进入 AVFrame
6- 将 AVFrame 编码成 AVPacket
7-写数据包
8- 转到 1
我在http://dranger.com/ffmpeg/tutorial01.html 中阅读了danger 教程，还使用了decode_encoding 示例。我可以阅读视频、提取视频帧并将它们转换为 CV Mat。我的问题从将 cv Mat 转换为 AVFrame 并将其编码为 AVPacket 开始。
你能帮我解决这个问题吗？
这是我的代码：
int main(int argc, char **argv)
AVOutputFormat *ofmt = NULL;
AVFormatContext *ifmt_ctx = NULL, *ofmt_ctx = NULL;
AVPacket pkt;
AVCodecContext *pCodecCtx = NULL;
AVCodec *pCodec = NULL;
AVFrame *pFrame = NULL;
AVFrame *pFrameRGB = NULL;
int videoStream=-1;
int audiostream=-1;
int frameFinished;
int numBytes;
uint8_t *buffer = NULL;
struct SwsContext *sws_ctx = NULL;
FrameManipulation *mal_frame;
const char *in_filename, *out_filename;
int ret, i;
if (argc < 3)
printf("usage: %s input output\n"
"API example program to remux a media file with libavformat and libavcodec.\n"
"The output format is guessed according to the file extension.\n"
"\n", argv[0]);
return 1;
in_filename = arg[1];
out_filename = arg[2];
av_register_all();
if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0)
fprintf(stderr, "Could not open input file '%s'", in_filename);
goto end;
if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0)
fprintf(stderr, "Failed to retrieve input stream information");
goto end;
av_dump_format(ifmt_ctx, 0, in_filename, 0);
avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, out_filename);
if (!ofmt_ctx)
fprintf(stderr, "Could not create output context\n");
ret = AVERROR_UNKNOWN;
goto end;
ofmt = ofmt_ctx->oformat;
for (i = 0; i < ifmt_ctx->nb_streams; i++)
AVStream *in_stream = ifmt_ctx->streams[i];
AVStream *out_stream = avformat_new_stream(ofmt_ctx, in_stream->codec->codec);
if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO &&
videoStream < 0)
videoStream=i;
if(ifmt_ctx->streams[i]->codec->codec_type==AVMEDIA_TYPE_AUDIO &&
audioStream < 0)
audioStream=i;
if (!out_stream)
fprintf(stderr, "Failed allocating output stream\n");
ret = AVERROR_UNKNOWN;
goto end;
ret = avcodec_copy_context(out_stream->codec, in_stream->codec);
if (ret < 0)
fprintf(stderr, "Failed to copy context from input to output stream codec context\n");
goto end;
out_stream->codec->codec_tag = 0;
if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
out_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
pCodec=avcodec_find_decoder(ifmt_ctx->streams[videoStream]->codec->codec_id);
pCodecCtx = avcodec_alloc_context3(pCodec);
if(avcodec_copy_context(pCodecCtx, ifmt_ctx->streams[videoStream]->codec) != 0)
fprintf(stderr, "Couldn't copy codec context");
return -1; // Error copying codec context
// Open codec
if(avcodec_open2(pCodecCtx, pCodec, NULL)<0)
return -1; // Could not open codec
// Allocate video frame
pFrame=av_frame_alloc();
// Allocate an AVFrame structure
pFrameRGB=av_frame_alloc();
// Determine required buffer size and allocate buffer
numBytes=avpicture_get_size(AV_PIX_FMT_RGB24, ifmt_ctx->streams[videoStream]->codec->width,
ifmt_ctx->streams[videoStream]->codec->height);
buffer=(uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
// Assign appropriate parts of buffer to image planes in pFrameRGB
// Note that pFrameRGB is an AVFrame, but AVFrame is a superset
// of AVPicture
avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24,
ifmt_ctx->streams[videoStream]->codec->width, ifmt_ctx->streams[videoStream]->codec->height);
av_dump_format(ofmt_ctx, 0, out_filename, 1);
if (!(ofmt->flags & AVFMT_NOFILE))
ret = avio_open(&ofmt_ctx->pb, out_filename, AVIO_FLAG_WRITE);
if (ret < 0)
fprintf(stderr, "Could not open output file '%s'", out_filename);
goto end;
ret = avformat_write_header(ofmt_ctx, NULL);
if (ret < 0)
fprintf(stderr, "Error occurred when opening output file\n");
goto end;
// Assign appropriate parts of buffer to image planes in pFrameRGB
// Note that pFrameRGB is an AVFrame, but AVFrame is a superset
// of AVPicture
avpicture_fill((AVPicture *)pFrameRGB, buffer, AV_PIX_FMT_BGR24,
ifmt_ctx->streams[videoStream]->codec->width,
ifmt_ctx->streams[videoStream]->codec->height);
// initialize SWS context for software scaling
sws_ctx = sws_getContext(
ifmt_ctx->streams[videoStream]->codec->width,
ifmt_ctx->streams[videoStream]->codec->height,
ifmt_ctx->streams[videoStream]->codec->pix_fmt,
ifmt_ctx->streams[videoStream]->codec->width,
ifmt_ctx->streams[videoStream]->codec->height,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL,
NULL,
NULL
);
// Loop through packets
while (1)
AVStream *in_stream, *out_stream;
ret = av_read_frame(ifmt_ctx, &pkt);
if(pkt.stream_index==videoStream)
// Decode video frame
avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &pkt);
if(frameFinished)
sws_scale(sws_ctx, (uint8_t const * const *)pFrame->data,
pFrame->linesize, 0, pCodecCtx->height,
pFrameRGB->data, pFrameRGB->linesize);
cv::Mat img= mal_frame->process(
pFrameRGB,pFrame->width,pFrame->height);
/* My problem is Here ------------*/
avpicture_fill((AVPicture*)pFrameRGB,
img.data,
PIX_FMT_BGR24,
outStream->codec->width,
outStream->codec->height);
pFrameRGB->width = ifmt_ctx->streams[videoStream]->codec->width;
pFrameRGB->height = ifmt_ctx->streams[videoStream]->codec->height;
avcodec_encode_video2(ifmt_ctx->streams[videoStream]->codec ,
&pkt , pFrameRGB , &gotPacket);
/*
I get this error
[swscaler @ 0x14b58a0] bad src image pointers
[swscaler @ 0x14b58a0] bad src image pointers
*/
/* My Problem Ends here ---------- */
if (ret < 0)
break;
in_stream = ifmt_ctx->streams[pkt.stream_index];
out_stream = ofmt_ctx->streams[pkt.stream_index];
//log_packet(ifmt_ctx, &pkt, "in");
/* copy packet */
pkt.pts = av_rescale_q_rnd(pkt.pts, in_stream->time_base, out_stream->time_base,
AV_ROUND_NEAR_INF);
pkt.dts = av_rescale_q_rnd(pkt.dts, in_stream->time_base, out_stream->time_base, AV_ROUND_NEAR_INF);
pkt.duration = av_rescale_q(pkt.duration, in_stream->time_base, out_stream->time_base);
pkt.pos = -1;
log_packet(ofmt_ctx, &pkt, "out");
ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
if (ret < 0)
fprintf(stderr, "Error muxing packet\n");
break;
av_free_packet(&pkt);
av_write_trailer(ofmt_ctx);
end:
avformat_close_input(&ifmt_ctx);
/* close output */
if (ofmt_ctx && !(ofmt->flags & AVFMT_NOFILE))
avio_closep(&ofmt_ctx->pb);
avformat_free_context(ofmt_ctx);
if (ret < 0 && ret != AVERROR_EOF)
return 1;
return 0;
当我运行这段代码时，我在这部分得到了未知的致命错误：
/* My problem is Here ------------*/
avpicture_fill((AVPicture*)pFrameRGB,
img.data,
PIX_FMT_BGR24,
outStream->codec->width,
outStream->codec->height);
pFrameRGB->width = ifmt_ctx->streams[videoStream]->codec->width;
pFrameRGB->height = ifmt_ctx->streams[videoStream]->codec->height;
avcodec_encode_video2(ifmt_ctx->streams[videoStream]->codec ,
&pkt , pFrameRGB , &gotPacket);
/*
I get this error
[swscaler @ 0x14b58a0] bad src image pointers
[swscaler @ 0x14b58a0] bad src image pointers
*/
/* My Problem Ends here ---------- */
这是我想将 cv Mat 转换回 AVFrame 并将其编码为 AVPacket 的地方。感谢您的帮助。
【问题讨论】：
当opencv支持libav作为后端进行解码/编码时，为什么还要手动呢？ @Aram opencv VideoWriter 在编码视频中不包含音频流。我不想为我的视频处理添加额外的阶段。 您使用的是哪个版本的 ffmpeg/libav？和opencv版本？FrameManipulation 结构是什么？你必须给我们一个可验证的例子才能运行。而且我注意到您使用的是remuxing 示例，而不是decoding_encoding，也很旧，很多已弃用的API。
我使用 ffmpeg 2.8 和 opencv 3.2。 FrameManipulation 减去一些对象，它不会改变帧的颜色空间和大小。将其视为模糊。 @halfelf
【参考方案1】：
在阅读了一些示例、阅读源代码和人们提供的一些帮助之后，我设法让代码运行起来。我使用了转码和编码示例并将它们混合在一起。 Here is my code
以下是重点： 1- libswscale 应该用于将具有所需数据包格式的 AVFrame 转换为 openCV Mat。为此，我们定义了
struct SwsContext *sws_ctx = NULL;
sws_ctx = sws_getContext(pCodecCtx->width,
pCodecCtx->height,
pCodecCtx->pix_fmt,
pCodecCtx->width,
pCodecCtx->height,
AV_PIX_FMT_BGR24,
SWS_BICUBIC,
NULL,
NULL,
NULL
);
要将 opencv Mat 转换回 AVFrame，应再次使用 swscale 并将 opencv BGR 帧格式转换为 YUV。所以，我这样做：
struct SwsContext *sws_ctx_bgr_yuv = NULL;
sws_ctx_bgr_yuv = sws_getContext(pCodecCtx->width,
pCodecCtx->height,
AV_PIX_FMT_BGR24,
pCodecCtx->width,
pCodecCtx->height,
pCodecCtx->pix_fmt //AV_PIX_FMT_YUV420p
,0,0,NULL,NULL);
还有，这里是帧读取/解码/编码循环：
while (1)
if ((ret = av_read_frame(ifmt_ctx, &packet)) < 0)
break;
stream_index = packet.stream_index;
type = ifmt_ctx->streams[packet.stream_index]->codec->codec_type;
av_log(NULL, AV_LOG_DEBUG, "Demuxer gave frame of stream_index %u\n",
stream_index);
if (filter_ctx[stream_index].filter_graph)
av_log(NULL, AV_LOG_DEBUG, "Going to reencode&filter the frame\n");
frame = av_frame_alloc();
if (!frame)
ret = AVERROR(ENOMEM);
break;
av_packet_rescale_ts(&packet,
ifmt_ctx->streams[stream_index]->time_base,
ifmt_ctx->streams[stream_index]->codec->time_base);
dec_func = (type == AVMEDIA_TYPE_VIDEO) ? avcodec_decode_video2 :
avcodec_decode_audio4;
ret = dec_func(ifmt_ctx->streams[stream_index]->codec, frame,
&got_frame, &packet);
if (ret < 0)
av_frame_free(&frame);
av_log(NULL, AV_LOG_ERROR, "Decoding failed\n");
break;
if (got_frame)
if(stream_index==video_index)
sws_scale(sws_ctx, (uint8_t const * const *)frame->data,
frame->linesize, 0, pCodecCtx->height,
pFrameRGB->data, pFrameRGB->linesize);
/*------------------------------------------------------------------------
/* Frame converts to opencv Mat
/*------------------------------------------------------------------------*/
cv::Mat img(frame->height,frame->width,CV_8UC3,pFrameRGB->data[0]);
img=manipulate_image(img); //this is opencv Mat, do whatever you want, but don't change its dimensions and format
//manipulate_function can be considered as as simple as blurring
const int stride[] = img.step[0] ;
/* opencv Mat converts back to AVFrame */
sws_scale(sws_ctx_bgr_yuv, &img.data, stride, 0, img.rows, frame->data, frame->linesize);
frame->pts = av_frame_get_best_effort_timestamp(frame);
/* AVFrame re-encodes to AVPacket and will be sent to encoder */
ret = filter_encode_write_frame(frame, stream_index);
av_frame_free(&frame);
if (ret < 0)
goto end;
else
av_frame_free(&frame);
else
/* remux this frame without reencoding */
av_packet_rescale_ts(&packet,
ifmt_ctx->streams[stream_index]->time_base,
ofmt_ctx->streams[stream_index]->time_base);
ret = av_interleaved_write_frame(ofmt_ctx, &packet);
if (ret < 0)
goto end;
av_free_packet(&packet);
【讨论】：
如果可能，将转换代码添加到您的答案中，而不仅仅是链接它。 @Micka 你的意思是全部吗？我的意思是不要让这个线程比现在更长。 @DavoodFalahati 相关的东西。如果那是648，那就这样吧。这比有人在几年内需要这个答案，单击链接并指向 404 错误要好。【参考方案2】：这是另一种使用 c++ 在 cv::Mat 和 AVframe 之间转换的方法，基于我发现并调试的一些 code。请注意它专门用于 8 位 3 通道图像，但可以通过在两个函数中更改 AV_PIX_FMT_BGR24 来更改。
希望这会有所帮助。
AVFrame cvmat_to_avframe(cv::Mat* frame)
AVFrame dst;
cv::Size frameSize = frame->size();
AVCodec *encoder = avcodec_find_encoder(AV_CODEC_ID_RAWVIDEO);
AVFormatContext* outContainer = avformat_alloc_context();
AVStream *outStream = avformat_new_stream(outContainer, encoder);
avcodec_get_context_defaults3(outStream->codec, encoder);
outStream->codec->pix_fmt = AV_PIX_FMT_BGR24;
outStream->codec->width = frame->cols;
outStream->codec->height = frame->rows;
avpicture_fill((AVPicture*)&dst, frame->data, AV_PIX_FMT_BGR24, outStream->codec->width, outStream->codec->height);
dst.width = frameSize.width;
dst.height = frameSize.height;
return dst;
cv::Mat avframe_to_cvmat(AVFrame *frame)
AVFrame dst;
cv::Mat m;
memset(&dst, 0, sizeof(dst));
int w = frame->width, h = frame->height;
m = cv::Mat(h, w, CV_8UC3);
dst.data[0] = (uint8_t *)m.data;
avpicture_fill( (AVPicture *)&dst, dst.data[0], AV_PIX_FMT_BGR24, w, h);
struct SwsContext *convert_ctx=NULL;
enum AVPixelFormat src_pixfmt = AV_PIX_FMT_BGR24;
enum AVPixelFormat dst_pixfmt = AV_PIX_FMT_BGR24;
convert_ctx = sws_getContext(w, h, src_pixfmt, w, h, dst_pixfmt,
SWS_FAST_BILINEAR, NULL, NULL, NULL);
sws_scale(convert_ctx, frame->data, frame->linesize, 0, h,
dst.data, dst.linesize);
sws_freeContext(convert_ctx);
return m;
【讨论】：
以上是关于Libav AVFrame 到 Opencv Mat 到 AVPacket 的转换的主要内容，如果未能解决你的问题，请参考以下文章