在 iOS 中使用 glsl 将 yv12 转换为 rgb，附上结果图像
Posted
技术标签:
【中文标题】在 iOS 中使用 glsl 将 yv12 转换为 rgb，附上结果图像【英文标题】：yv12 to rgb using glsl in iOS ,result image attached 【发布时间】：2012-06-19 00:57:25 【问题描述】：referred to this question
我使用 glsl 着色器将 yv12 帧数据转换为 rgb 数据，原始图像如下：
但结果图片与前者不一样，附如下：
以下是我将三个平面数据上传到纹理的代码：
- (GLuint) textureY: (Byte*)imageData
widthType: (int) width
heightType: (int) height
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
- (GLuint) textureU: (Byte*)imageData
widthType: (int) width
heightType: (int) height
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
- (GLuint) textureV: (Byte*)imageData
widthType: (int) width
heightType: (int) height
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
- (void) readYUVFile
NSString *file = [[NSBundle mainBundle] pathForResource:@"video" ofType:@"yv12"];
NSLog(@"%@",file);
NSData* fileData = [NSData dataWithContentsOfFile:file];
//NSLog(@"%@",[fileData description]);
NSInteger width = 352;
NSInteger height = 288;
NSInteger uv_width = width / 2;
NSInteger uv_height = height / 2;
NSInteger dataSize = [fileData length];
NSLog(@"%i\n",dataSize);
GLint nYsize = width * height;
GLint nUVsize = uv_width * uv_height;
GLint nCbOffSet = nYsize;
GLint nCrOffSet = nCbOffSet + nUVsize;
Byte *spriteData = (Byte *)malloc(dataSize);
[fileData getBytes:spriteData length:dataSize];
Byte* uData = spriteData + nCbOffSet;
//NSLog(@"%@\n",[[NSData dataWithBytes:uData length:nUVsize] description]);
Byte* vData = spriteData + nCrOffSet;
//NSLog(@"%@\n",[[NSData dataWithBytes:vData length:nUVsize] description]);
/**
Byte *YPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<nYsize; i++)
YPlanarData[i]= spriteData[i];
Byte *UPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++)
for (int j=0; j<width; j++)
int numInUVsize = (i/2)*uv_width+j/2;
UPlanarData[i*width+j]=uData[numInUVsize];
Byte *VPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++)
for (int j=0; j<width; j++)
int numInUVsize = (i/2)*uv_width+j/2;
VPlanarData[i*width+j]=vData[numInUVsize];
**/
_YPlanarTexture = [self textureY:spriteData widthType:width heightType:height];
_UPlanarTexture = [self textureU:uData widthType:uv_width heightType:uv_height];
_VPlanarTexture = [self textureV:vData widthType:uv_width heightType:uv_height];
free(spriteData);
和我的片段着色器代码：
precision highp float;
uniform sampler2D SamplerY;
uniform sampler2D SamplerU;
uniform sampler2D SamplerV;
varying highp vec2 coordinate;
void main()
highp vec3 yuv,yuv1;
highp vec3 rgb;
yuv.x = texture2D(SamplerY, coordinate).r;
yuv.y = texture2D(SamplerU, coordinate).r-0.5;
yuv.z = texture2D(SamplerV, coordinate).r-0.5 ;
rgb = mat3( 1, 1, 1,
0, -.34414, 1.772,
1.402, -.71414, 0) * yuv;
gl_FragColor = vec4(rgb, 1);
我的困惑是转换公式 而我使用这个公式直接将yv12数据转换为rgb24，并用
CGImageCreate(iwidth,
iheight,
8,
24,
iwidth*3,
colorSpace,
bitmapInfo,
provider,
NULL,
NO,
kCGRenderingIntentDefault);
结果图像是正确的。 但是使用着色器（在ios设备上运行的直接变换方法是转储）转向这个问题，我尝试了一些技巧（将UV刨床扩展到（2 * uv_width）* 2（uv_height）矩形，然后上传纹理)，但在同一张更红的图像中失败了。
如何解决这个问题？
附上我的整个 glView.m 代码：
#import "OpenGLView.h"
typedef struct
float Position[3];
float TexCoord[2];
Vertex;
const Vertex Vertices[] =
1, -1, 0,1,1,
1, 1, 0,1,0,
-1, 1, 0,0,0,
-1, -1, 0,0,1
;
const GLubyte Indices[] =
0, 1, 2,
2, 3, 0
;
@interface OpenGLView ()
- (void)setupLayer;
- (void)setupContext;
- (void)setupRenderBuffer;
- (void)setupFrameBuffer;
- (void)render;
- (GLuint)compileShader:(NSString*)shaderName withType:(GLenum)shaderType;
- (void)setupVBOs;
- (void)compileShaders;
- (void) readYUVFile;
@end
@implementation OpenGLView
- (void)setupVBOs
GLuint vertexBuffer;
glGenBuffers(1, &vertexBuffer);
glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer);
glBufferData(GL_ARRAY_BUFFER, sizeof(Vertices), Vertices, GL_STATIC_DRAW);
GLuint indexBuffer;
glGenBuffers(1, &indexBuffer);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indexBuffer);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(Indices), Indices, GL_STATIC_DRAW);
- (id)initWithFrame:(CGRect)frame
self = [super initWithFrame:frame];
if (self)
// Initialization code[]
self.backgroundColor = [UIColor redColor];
[self setupLayer];
[self setupContext];
[self setupRenderBuffer];
[self setupFrameBuffer];
[self setupVBOs];
[self compileShaders];
[self readYUVFile];
[self render];
return self;
+ (Class)layerClass
return [CAEAGLLayer class];
-(void)setupLayer
_eaglLayer = (CAEAGLLayer *)self.layer;
_eaglLayer.opaque = YES;
- (void)setupContext
EAGLRenderingAPI api = kEAGLRenderingAPIOpenGLES2;
_context = [[[EAGLContext alloc] initWithAPI:api] autorelease];
if (!_context)
NSLog(@"Failed to initialize OpenGLES 2.0 context");
exit(1);
if (![EAGLContext setCurrentContext:_context])
NSLog(@"Failed to set current OpenGL context");
exit(1);
- (void)setupRenderBuffer
glGenRenderbuffers(1, &_colorRenderBuffer);
glBindRenderbuffer(GL_RENDERBUFFER, _colorRenderBuffer);
[_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:_eaglLayer];
- (void)setupFrameBuffer
GLuint framebuffer;
glGenFramebuffers(1, &framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_RENDERBUFFER, _colorRenderBuffer);
- (GLuint) textureY: (Byte*)imageData
widthType: (int) width
heightType: (int) height
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
- (GLuint) textureU: (Byte*)imageData
widthType: (int) width
heightType: (int) height
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_RED_EXT, width, height, 0, GL_RED_EXT, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
- (GLuint) textureV: (Byte*)imageData
widthType: (int) width
heightType: (int) height
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_RED_EXT, width, height, 0, GL_RED_EXT, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
- (void) readYUVFile
NSString *file = [[NSBundle mainBundle] pathForResource:@"video" ofType:@"yv12"];
NSLog(@"%@",file);
NSData* fileData = [NSData dataWithContentsOfFile:file];
//NSLog(@"%@",[fileData description]);
NSInteger width = 352;
NSInteger height = 288;
NSInteger uv_width = width / 2;
NSInteger uv_height = height / 2;
NSInteger dataSize = [fileData length];
NSLog(@"%i\n",dataSize);
GLint nYsize = width * height;
GLint nUVsize = uv_width * uv_height;
GLint nCbOffSet = nYsize;
GLint nCrOffSet = nCbOffSet + nUVsize;
Byte *spriteData = (Byte *)malloc(dataSize);
[fileData getBytes:spriteData length:dataSize];
Byte* uData = spriteData + nCbOffSet;
//NSLog(@"%@\n",[[NSData dataWithBytes:uData length:nUVsize] description]);
Byte* vData = spriteData + nCrOffSet;
//NSLog(@"%@\n",[[NSData dataWithBytes:vData length:nUVsize] description]);
Byte *YPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<nYsize; i++)
YPlanarData[i]= spriteData[i];
Byte *UPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++)
for (int j=0; j<width; j++)
int numInUVsize = (i/2)*uv_width+j/2;
UPlanarData[i*width+j]=uData[numInUVsize];
Byte *VPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++)
for (int j=0; j<width; j++)
int numInUVsize = (i/2)*uv_width+j/2;
VPlanarData[i*width+j]=vData[numInUVsize];
_YPlanarTexture = [self textureY:YPlanarData widthType:width heightType:height];
_UPlanarTexture = [self textureU:UPlanarData widthType:width heightType:height];
_VPlanarTexture = [self textureV:VPlanarData widthType:width heightType:height];
free(spriteData);
- (void)render
glClearColor(0,0,0 , 1.0);
glClear(GL_COLOR_BUFFER_BIT);
// 1
glViewport(0, 200, self.frame.size.width, 558);
// 2
glVertexAttribPointer(_positionSlot, 3, GL_FLOAT, GL_FALSE,
sizeof(Vertex), 0);
glVertexAttribPointer(_texCoordSlot, 2, GL_FLOAT, GL_FALSE,
sizeof(Vertex), (GLvoid*) (sizeof(float) *3));
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, _YPlanarTexture);
glUniform1i(_textureUniformY, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, _UPlanarTexture);
glUniform1i(_textureUniformU, 1);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, _VPlanarTexture);
glUniform1i(_textureUniformV, 2);
// 3
glDrawElements(GL_TRIANGLES, sizeof(Indices)/sizeof(Indices[0]),
GL_UNSIGNED_BYTE, 0);
[_context presentRenderbuffer:GL_RENDERBUFFER];
/*
// Only override drawRect: if you perform custom drawing.
// An empty implementation adversely affects performance during animation.
- (void)drawRect:(CGRect)rect
// Drawing code
*/
- (GLuint)compileShader:(NSString*)shaderName withType:(GLenum)shaderType
// 1
NSString* shaderPath = [[NSBundle mainBundle] pathForResource:shaderName
ofType:@"glsl"];
NSError* error;
NSString* shaderString = [NSString stringWithContentsOfFile:shaderPath
encoding:NSUTF8StringEncoding error:&error];
if (!shaderString)
NSLog(@"Error loading shader: %@", error.localizedDescription);
exit(1);
// 2
GLuint shaderHandle = glCreateShader(shaderType);
// 3
const char* shaderStringUTF8 = [shaderString UTF8String];
int shaderStringLength = [shaderString length];
glShaderSource(shaderHandle, 1, &shaderStringUTF8, &shaderStringLength);
// 4
glCompileShader(shaderHandle);
// 5
GLint compileSuccess;
glGetShaderiv(shaderHandle, GL_COMPILE_STATUS, &compileSuccess);
if (compileSuccess == GL_FALSE)
GLchar messages[256];
glGetShaderInfoLog(shaderHandle, sizeof(messages), 0, &messages[0]);
NSString *messageString = [NSString stringWithUTF8String:messages];
NSLog(@"%@", messageString);
exit(1);
return shaderHandle;
- (void)compileShaders
// 1
GLuint vertexShader = [self compileShader:@"SimpleVertex"
withType:GL_VERTEX_SHADER];
GLuint fragmentShader = [self compileShader:@"SimpleFragment"
withType:GL_FRAGMENT_SHADER];
// 2
GLuint programHandle = glCreateProgram();
glAttachShader(programHandle, vertexShader);
glAttachShader(programHandle, fragmentShader);
glLinkProgram(programHandle);
// 3
GLint linkSuccess;
glGetProgramiv(programHandle, GL_LINK_STATUS, &linkSuccess);
if (linkSuccess == GL_FALSE)
GLchar messages[256];
glGetProgramInfoLog(programHandle, sizeof(messages), 0, &messages[0]);
NSString *messageString = [NSString stringWithUTF8String:messages];
NSLog(@"%@", messageString);
exit(1);
// 4
glUseProgram(programHandle);
// 5
_positionSlot = glGetAttribLocation(programHandle, "position");
glEnableVertexAttribArray(_positionSlot);
_texCoordSlot = glGetAttribLocation(programHandle, "textureCoordinate");
glEnableVertexAttribArray(_texCoordSlot);
_YPlanarTexture = glGetUniformLocation(programHandle, "SamplerY");
_UPlanarTexture = glGetUniformLocation(programHandle, "SamplerU");
_VPlanarTexture = glGetUniformLocation(programHandle, "SamplerV");
@end
【问题讨论】：
我的着色器看起来不对。 yuv.y 和 yuv.z 都在拉红色通道，而他们应该拉动绿色和蓝色通道，不是吗？ 谢谢，上传纹理时，三个平面数据我都使用 GL_RED_EXT 作为内部格式，所以在着色器中我使用 .r 获取 yuv 值，我错了吗？我可以纠正吗？请给我详细的解决方案。 您介意粘贴 SimpleVertex.glsl 吗？谢谢！ 属性 vec4 位置；属性 mediump vec2 textureCoordinate;改变 mediump vec2 坐标； void main() gl_Position = 位置；坐标=纹理坐标； 【参考方案1】：我的愚蠢错误，三个上传纹理过程和片段着色器是正确的，但以下代码不兼容：
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, _YPlanarTexture);
glUniform1i(_textureUniformY, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, _UPlanarTexture);
glUniform1i(_textureUniformU, 1);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, _VPlanarTexture);
glUniform1i(_textureUniformV, 2);
及以下：
_YPlanarTexture = glGetUniformLocation(programHandle, "SamplerY");
_UPlanarTexture = glGetUniformLocation(programHandle, "SamplerU");
_VPlanarTexture = glGetUniformLocation(programHandle, "SamplerV");
所以用这个替换：
_textureUniformY = glGetUniformLocation(programHandle, "SamplerY");
_textureUniformU = glGetUniformLocation(programHandle, "SamplerU");
_textureUniformV = glGetUniformLocation(programHandle, "SamplerV");
那么它就会做正确的事情。
【讨论】：
以上是关于在 iOS 中使用 glsl 将 yv12 转换为 rgb，附上结果图像的主要内容，如果未能解决你的问题，请参考以下文章