如何从 NSImage/UIImage 获取 AVFrame(ffmpeg)

Posted 2023-02-16

【中文标题】如何从 NSImage/UIImage 获取 AVFrame(ffmpeg)

【英文标题】：How to get AVFrame(ffmpeg) from NSImage/UIImage

【发布时间】：2010-06-25 17:09:00

【问题描述】：

我想将 NSImage/UIImage 转换为 AVFrame(ffmpeg)。 我找到了一个示例代码。 http://lists.mplayerhq.hu/pipermail/libav-user/2010-April/004550.html 但是这段代码不起作用。

我尝试了另一种方法。

  AVFrame *frame = avcodec_alloc_frame();

  int numBytes = avpicture_get_size(PIX_FMT_YUV420P, outputWidth, outputHeight);
  uint8_t *buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
  avpicture_fill((AVPicture *)frame, buffer, PIX_FMT_YUV420P, outputWidth, outputHeight);


  //UIImage *image = … smothing … ;
  NSImage *image = … smothing … ;

  //CGImageRef newCgImage = image.CGImage;
  CGImageRef newCgImage = [image CGImageForProposedRect:nil context:nil hints:nil];
  //NSBitmapImageRep* bm = [NSBitmapImageRep imageRepWithData:[image TIFFRepresentation]];
  //CGImageRef newCgImage = [bm CGImage];

  size_t w = CGImageGetWidth(newCgImage);
  size_t h = CGImageGetHeight(cgImage);

  CGDataProviderRef dataProvider = CGImageGetDataProvider(newCgImage);
  CFDataRef bitmapData = CGDataProviderCopyData(dataProvider);
  uint8_t *buffer = (uint8_t *)CFDataGetBytePtr(bitmapData);

  frame->linesize[0] = w;
  int y, x;
  for (y = 0; y < h; y++) 
    for (x = 0; x < w; x++) 
      int z = y * w + x;
      frame->data[0][z] = buffer[z];
    
  

但是这个 AVFrame 给了我绿色的图片。

请告诉我如何获得它。

谢谢。

以下是额外的。

我再次尝试注意颜色格式。 我找到了将 RGB 转换为 YUM 的示例。 How to perform RGB->YUV conversion in C/C++?

新代码是这样的。但是，还是不行……

#import <Foundation/Foundation.h>
#import <AppKit/AppKit.h>
#import <libavutil/avstring.h>
#import <libavcodec/avcodec.h>
#import <libavformat/avformat.h>
#import <libswscale/swscale.h>

int main(int argc, char *argv[]) 
  NSAutoreleasePool* pool = [[NSAutoreleasePool alloc] init];

  int i;
  int outputWidth  = 480; //size must be same size of arg
  int outputHeight = 360; //size must be same size of arg

  av_register_all();

  AVOutputFormat *format = av_guess_format("mp4", NULL, NULL);
  if(!format) return -1;

  AVFormatContext *outFormatCtx = avformat_alloc_context();
  if(!outFormatCtx) return -1;
  outFormatCtx->oformat  = format;
  av_strlcpy(outFormatCtx->filename, "test.mov", sizeof(outFormatCtx->filename));

  AVStream *vstream = av_new_stream(outFormatCtx, 0);
  if(!vstream) return -1;

  enum CodecID codec_id = av_guess_codec(outFormatCtx->oformat,
                       NULL,
                       outFormatCtx->filename,
                       NULL, CODEC_TYPE_VIDEO);

  AVCodec *ovCodec = avcodec_find_encoder(codec_id);
  if(!ovCodec) return -1;

  AVCodecContext *ovCodecCtx  = vstream->codec;
  ovCodecCtx->codec_id = ovCodec->id;
  ovCodecCtx->codec_type = CODEC_TYPE_VIDEO;
  ovCodecCtx->width = outputWidth;
  ovCodecCtx->height = outputHeight;

  ovCodecCtx->pix_fmt = PIX_FMT_NONE;
  if(ovCodec && ovCodec->pix_fmts)
    const enum PixelFormat *p = ovCodec->pix_fmts;
    while(*p++ != -1)
      if(*p == ovCodecCtx->pix_fmt) break;
    
    if(*p == -1) ovCodecCtx->pix_fmt = ovCodec->pix_fmts[0];
  

  ovCodecCtx->time_base.num = 1;
  ovCodecCtx->time_base.den = 30;

  if(format->flags & AVFMT_GLOBALHEADER)
    ovCodecCtx->flags |= CODEC_FLAG_GLOBAL_HEADER;

  if(avcodec_open(ovCodecCtx, ovCodec) != 0) return -1;

  if (! ( format->flags & AVFMT_NOFILE )) 
    if(url_fopen(&outFormatCtx->pb, outFormatCtx->filename, URL_WRONLY) < 0) return NO;
  

  av_write_header(outFormatCtx);

  int buf_size = ovCodecCtx->width * ovCodecCtx->height * 4;
  uint8_t *buf = av_malloc(buf_size);

  AVFrame *buffer_frame = avcodec_alloc_frame();
  if(!buffer_frame) return -1;

  AVFrame *frame = avcodec_alloc_frame();
  if(!frame) return -1;

  int numBytes = avpicture_get_size(PIX_FMT_YUV420P, outputWidth, outputHeight);
  uint8_t *buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t));
  avpicture_fill((AVPicture *)frame, buffer, PIX_FMT_YUV420P, outputWidth, outputHeight);

  for(i=1;i<argc;i++)
    NSAutoreleasePool *innerPool = [[NSAutoreleasePool alloc] init];

    NSImage *image = [[NSImage alloc] initWithContentsOfFile:[NSString stringWithCString: argv[i] encoding: NSUTF8StringEncoding]];
    CGImageRef imageRef = [image CGImageForProposedRect:nil context:nil hints:nil];

    size_t w = CGImageGetWidth(imageRef);
    size_t h = CGImageGetHeight(imageRef);
    size_t bytesPerRow = CGImageGetBytesPerRow(imageRef);

    CGDataProviderRef dataProvider = CGImageGetDataProvider(imageRef);
    CFDataRef bitmapData = CGDataProviderCopyData(dataProvider);
    uint8_t *buff = (uint8_t *)CFDataGetBytePtr(bitmapData);


    uint8_t R,G,B,Y,U,V;
    int x,y;
    for(y=0;y<h;y++)
      for(x=0;x<w;x++)
        uint8_t *tmp = buff + y * bytesPerRow + x * 4;
        R = *(tmp + 3);
        G = *(tmp + 2);
        B = *(tmp + 1);

        Y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16;
        U =  -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128;
        V =  (0.439 * R) - (0.368 * G) - (0.071 * B) + 128;
//printf("y:%d x:%d  R:%d,G:%d,B:%d  Y:%d,U:%d,V:%d \n",y,x,R,G,B,Y,U,V);
        frame->data[0][y*frame->linesize[0]+x]= Y;
        //frame->data[1][y*frame->linesize[1]+x]= U; //if coment out "Bus error"
        //frame->data[2][y*frame->linesize[2]+x]= V; //if coment out "Bus error" 
      
    

    CGImageRelease(imageRef);
    CFRelease(bitmapData);

    int out_size = avcodec_encode_video (ovCodecCtx, buf, buf_size, frame);

    AVPacket outPacket;
    av_init_packet(&outPacket);

    outPacket.stream_index= vstream->index;
    outPacket.data= buf;
    outPacket.size= out_size;

    //outPacket.pts = ?;
    //outPacket.dts = ?;

    if(ovCodecCtx->coded_frame->key_frame)
      outPacket.flags |= PKT_FLAG_KEY;

    if(av_interleaved_write_frame(outFormatCtx, &outPacket) != 0) return -1;

    [image release];

    [innerPool release];
  

  av_write_trailer(outFormatCtx);
  if (! ( format->flags & AVFMT_NOFILE ))
    if(url_fclose(outFormatCtx->pb) < 0) return -1;

  avcodec_close(vstream->codec);

  for(i = 0; i < outFormatCtx->nb_streams; i++) 
    av_freep(&outFormatCtx->streams[i]->codec);
    av_freep(&outFormatCtx->streams[i]);
  
  av_freep(&outFormatCtx);

  av_free(buffer);
  av_free(frame);
  av_free(buffer_frame);

  [pool release];
  return 0;


and mekefile is like this.

CC = /usr/bin/gcc 
CFLAGS  = -O4 -Wall -I/usr/local/include
LDFLAGS =
LDLIBS  = -L/usr/local/bin -lavutil -lavformat -lavcodec -lswscale
FRAMEWORK = -framework Foundation -framework AppKit #-framework CoreGraphics

OBJS = test.o 

test: $(OBJS)
    $(CC) -o $@ $(LDFLAGS) $(OBJS) $(LDLIBS) $(FRAMEWORK) -lz  -lbz2 -arch x86_64

请有人帮助我。

【参考方案1】：

CGImage 和目标 AVFrame 的数据之间存在色彩空间不匹配。为了解决这个问题，您需要将 CGImage 数据（可能是 ARGB）转换为 YUV420 格式（FFMpeg 具有内置格式转换器）。您可以使用CGImageGetBitsPerComponent、CGImageGetBitsPerPixel 和CGImageGetBytesPerRow 函数获取有关 CGImage 色彩空间的信息。

【讨论】：

感谢您的回复，Laurent Etiemble。但我仍然无法转换。你能给我更多信息吗？如果你给我一些代码，我很感激你。谢谢。