Opengl es 2.0 在视频上绘制位图叠加
Posted
技术标签:
【中文标题】Opengl es 2.0 在视频上绘制位图叠加【英文标题】：Opengl es 2.0 draw bitmap overlay on video 【发布时间】：2017-01-28 02:12:48 【问题描述】：我正在尝试在视频的每一帧上绘制位图作为叠加层。我找到了一个关于如何解码和编码视频的示例，它正在工作。此示例有一个带有 drawFrame 函数的 TextureRenderer 类，我需要修改该函数才能添加位图。我是opengl的新手，但我了解到我需要使用位图创建纹理并绑定它。我在下面的代码中尝试过，但它抛出了异常。
/*
* Copyright (C) 2013 The android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// from: https://android.googlesource.com/platform/cts/+/lollipop-release/tests/tests/media/src/android/media/cts/TextureRender.java
// blob: 4125dcfcfed6ed7fddba5b71d657dec0d433da6a
// modified: removed unused method bodies
// modified: use GL_LINEAR for GL_TEXTURE_MIN_FILTER to improve quality.
package com.example.name.videoeditortest;
/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import android.opengl.Matrix;
import android.util.Log;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
class TextureRender
private Bitmap bitmap;
private static final String TAG = "TextureRender";
private static final int FLOAT_SIZE_BYTES = 4;
private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private final float[] mTriangleVerticesData =
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
;
private FloatBuffer mTriangleVertices;
private static final String VERTEX_SHADER =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() \n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"\n";
private static final String FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() \n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"\n";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int mProgram;
private int mTextureID = -12345;
private int mTextureBitmapID = -12345;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
public TextureRender()
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
public int getTextureId()
return mTextureID;
public void drawFrame(SurfaceTexture st)
checkGlError("onDrawFrame start");
st.getTransformMatrix(mSTMatrix);
GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glUseProgram(mProgram);
checkGlError("glUseProgram");
//Bing textrues
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
GLES20.glActiveTexture(GLES20.GL_TEXTURE_2D);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureBitmapID);
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maPosition");
GLES20.glEnableVertexAttribArray(maPositionHandle);
checkGlError("glEnableVertexAttribArray maPositionHandle");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer maTextureHandle");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray maTextureHandle");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays");
GLES20.glFinish();
/**
* Initializes GL state. Call this after the EGL surface has been created and made current.
*/
public void surfaceCreated()
mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
if (mProgram == 0)
throw new RuntimeException("failed creating program");
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
checkGlError("glGetAttribLocation aPosition");
if (maPositionHandle == -1)
throw new RuntimeException("Could not get attrib location for aPosition");
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
checkGlError("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1)
throw new RuntimeException("Could not get attrib location for aTextureCoord");
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
checkGlError("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1)
throw new RuntimeException("Could not get attrib location for uMVPMatrix");
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
checkGlError("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1)
throw new RuntimeException("Could not get attrib location for uSTMatrix");
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
mTextureID = textures[0];
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError("glBindTexture mTextureID");
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE);
checkGlError("glTexParameter");
mTextureBitmapID = loadBitmapTexture();
private int loadBitmapTexture()
final int[] textureHandle = new int[1];
GLES20.glGenTextures(1, textureHandle, 0);
if (textureHandle[0] != 0)
// Bind to the texture in OpenGL
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureHandle[0]);
// Set filtering
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
// Load the bitmap into the bound texture.
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);
if (textureHandle[0] == 0)
throw new RuntimeException("Error loading texture.");
return textureHandle[0];
/**
* Replaces the fragment shader.
*/
public void changeFragmentShader(String fragmentShader)
GLES20.glDeleteProgram(mProgram);
mProgram = createProgram(VERTEX_SHADER, fragmentShader);
if (mProgram == 0)
throw new RuntimeException("failed creating program");
private int loadShader(int shaderType, String source)
int shader = GLES20.glCreateShader(shaderType);
checkGlError("glCreateShader type=" + shaderType);
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0)
Log.e(TAG, "Could not compile shader " + shaderType + ":");
Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
return shader;
private int createProgram(String vertexSource, String fragmentSource)
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if (vertexShader == 0)
return 0;
int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if (pixelShader == 0)
return 0;
int program = GLES20.glCreateProgram();
checkGlError("glCreateProgram");
if (program == 0)
Log.e(TAG, "Could not create program");
GLES20.glAttachShader(program, vertexShader);
checkGlError("glAttachShader");
GLES20.glAttachShader(program, pixelShader);
checkGlError("glAttachShader");
GLES20.glLinkProgram(program);
int[] linkStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
if (linkStatus[0] != GLES20.GL_TRUE)
Log.e(TAG, "Could not link program: ");
Log.e(TAG, GLES20.glGetProgramInfoLog(program));
GLES20.glDeleteProgram(program);
program = 0;
return program;
public void checkGlError(String op)
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR)
Log.e(TAG, op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
public void setBitmap(Bitmap bitmap)
this.bitmap = bitmap;
/**
* Saves the current frame to disk as a PNG image. Frame starts from (0,0).
* <p>
* Useful for debugging.
*/
public static void saveFrame(String filename, int width, int height)
// glReadPixels gives us a ByteBuffer filled with what is essentially big-endian RGBA
// data (i.e. a byte of red, followed by a byte of green...). We need an int[] filled
// with native-order ARGB data to feed to Bitmap.
//
// If we implement this as a series of buf.get() calls, we can spend 2.5 seconds just
// copying data around for a 720p frame. It's better to do a bulk get() and then
// rearrange the data in memory. (For comparison, the PNG compress takes about 500ms
// for a trivial frame.)
//
// So... we set the ByteBuffer to little-endian, which should turn the bulk IntBuffer
// get() into a straight memcpy on most Android devices. Our ints will hold ABGR data.
// Swapping B and R gives us ARGB. We need about 30ms for the bulk get(), and another
// 270ms for the color swap.
//
// Making this even more interesting is the upside-down nature of GL, which means we
// may want to flip the image vertically here.
ByteBuffer buf = ByteBuffer.allocateDirect(width * height * 4);
buf.order(ByteOrder.LITTLE_ENDIAN);
GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf);
buf.rewind();
int pixelCount = width * height;
int[] colors = new int[pixelCount];
buf.asIntBuffer().get(colors);
for (int i = 0; i < pixelCount; i++)
int c = colors[i];
colors[i] = (c & 0xff00ff00) | ((c & 0x00ff0000) >> 16) | ((c & 0x000000ff) << 16);
FileOutputStream fos = null;
try
fos = new FileOutputStream(filename);
Bitmap bmp = Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
bmp.compress(Bitmap.CompressFormat.PNG, 90, fos);
bmp.recycle();
catch (IOException ioe)
throw new RuntimeException("Failed to write file " + filename, ioe);
finally
try
if (fos != null) fos.close();
catch (IOException ioe2)
throw new RuntimeException("Failed to close file " + filename, ioe2);
Log.d(TAG, "Saved " + width + "x" + height + " frame as '" + filename + "'");
抛出异常：
E/ExtractDecodeEditEncodeMuxTest: error while releasing muxer
java.lang.IllegalStateException: Can't stop due to wrong state.
at android.media.MediaMuxer.stop(MediaMuxer.java:231)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest.extractDecodeEditEncodeMux(ExtractDecodeEditEncodeMuxTest.java 434)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest.access$000(ExtractDecodeEditEncodeMuxTest.java:58)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest$TestWrapper.run(ExtractDecodeEditEncodeMuxTest.java:171)
at java.lang.Thread.run(Thread.java:841)
如果我在 drawFrame 中评论 GLES20.glActiveTexture(GLES20.GL_TEXTURE_2D);，视频会正确渲染，但没有位图。如果我在 drawFrame 中评论 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureBitmapID);，我会得到以下异常：
java.lang.RuntimeException: glVertexAttribPointer maPosition: glError 1280
at com.example.name.videoeditortest.TextureRender.checkGlError(TextureRender.java:259)
at com.example.name.videoeditortest.TextureRender.drawFrame(TextureRender.java:111)
at com.example.name.videoeditortest.OutputSurface.drawImage(OutputSurface.java:252)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest.doExtractDecodeEditEncodeMux(ExtractDecodeEditEncodeMuxTest.java:793)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest.extractDecodeEditEncodeMux(ExtractDecodeEditEncodeMuxTest.java:341)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest.access$000(ExtractDecodeEditEncodeMuxTest.java:58)
at com.example.name.videoeditortest.ExtractDecodeEditEncodeMuxTest$TestWrapper.run(ExtractDecodeEditEncodeMuxTest.java:171)
at java.lang.Thread.run(Thread.java:841)
【问题讨论】：
【参考方案1】：我发现有两件事我觉得不对。
您正试图同时绑定所有内容，并希望调用GLES20.glDrawArrays() 将绘制所有内容。
你只有一个着色器，你应该有两个：一个用于进行视频纹理渲染，另一个用于位图图层渲染。
您必须知道的是，可以通过多次调用 glDrawArrays 来绘制帧，每次调用只会在之前绘制的内容上绘制一小部分（基本上）。
在您的案例中渲染帧的第一部分应该是这样的：
# init
loadShaderForVideo()
loadShaderForBitmapLayer()
prepareYourArraysEtc()
...
#loop
GLClear()
updateVideoTexture()
drawFrame()
drawVideo()
drawBitmap()
drawVideo()
bindYourActiveTextureToVideo()
setYourVertexAttribAndUniform()
GLES20.glDrawArrays()
drawBitmap()
bindYourActiveTextureToBitmap()
setYourVertexAttribAndUniform() // This should be the same as above for video
// Considering you want to draw above your video, consider activating the blending for transparency :
GLES20.glEnable(GLES20.GL_BLEND);
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
GLES20.glDrawArrays()
关于着色器，看看这些：
两者都有一个共同的Vertex Shader：
public static final String vertexDefaultShaderCode =
"uniform mat4 uVPMatrix;" +
"uniform mat4 uModelMatrix;" + // uniform = input const
"attribute vec3 aPosition;" + // attribute = input property different for each vertex
"attribute vec2 aTexCoordinate;" +
"varying vec2 vTexCoordinate;" +// varying = output property different for each pixel
"void main() " +
"vTexCoordinate = aTexCoordinate;" +
"gl_Position = uVPMatrix * uModelMatrix * vec4(aPosition,1.0);" +
"";
然后是基本的fragment shader（用于您的位图 2D 纹理）：
public static final String fragmentDefaultShaderCode =
"precision mediump float;" +
"uniform sampler2D uTexture;" +
"varying vec2 vTexCoordinate;" +
"void main() " +
" gl_FragColor = texture2D(uTexture, vTexCoordinate);" +
"";
然后是视频渲染的不同版本：
public static final String fragmentExternalShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;" +
"uniform samplerExternalOES sTexture;" +
"varying vec2 vTexCoordinate;" +
"void main() " +
" gl_FragColor = texture2D(sTexture, vTexCoordinate);" +
"";
因此您将需要两个程序，一个带有 defaultVertexShader + defaultFragmentShader，另一个带有 defaultVertexShader + fragmentExternalShaderCode。
【讨论】：
感谢您的回答，真的很有帮助。如您所见，我说过我希望赏金赢家提供功能代码。再次感谢！ 是的，我看到了你的条件。如果我有太多空闲时间，我会更深入地了解这一点，但现在不能做更多。 你能分享完整的代码吗？这真的很有帮助。 嘿@ZiadHalabi 你能分享这个问题的更新代码吗！【参考方案2】：有缺陷但有效。根据 J.Jacobs-VP，您需要使用两个程序和两个着色器，一个用于视频帧，另一个用于位图。
/*
* Copyright (C) 2013 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var gles = 2
/**
* Code for rendering a texture onto a surface using OpenGL ES 2.0.
*/
internal class TextureRender
/** Video Frame **/
private val mTriangleVerticesData = floatArrayOf( // X, Y, Z, U, V
-1.0f, -1.0f, 0f, 0f, 0f,
1.0f, -1.0f, 0f, 1f, 0f,
-1.0f, 1.0f, 0f, 0f, 1f,
1.0f, 1.0f, 0f, 1f, 1f
)
private val mTriangleVertices: FloatBuffer
private val mMVPMatrix = FloatArray(16)
private val mSTMatrix = FloatArray(16)
private var mProgram = 0
var textureId = -12345
private set
private var muMVPMatrixHandle = 0
private var muSTMatrixHandle = 0
private var maPositionHandle = 0
private var maTextureHandle = 0
/** Bitmap Overlay **/
private val mBitmapTriangleVerticesData = floatArrayOf( // X, Y, Z, U, V
-1.0f, -1.0f, 0f, 0f, 0f,
1.0f, -1.0f, 0f, 1f, 0f,
-1.0f, 1.0f, 0f, 0f, 1f,
1.0f, 1.0f, 0f, 1f, 1f
)
private val mBitmapTriangleVertices: FloatBuffer
private var mBitmapProgram = 1
private val mBitmapMVPMatrix = FloatArray(16)
private val mBitmapSTMatrix = FloatArray(16)
private var mBitmapuMVPMatrixHandle = 0
private var mBitmapuSTMatrixHandle = 0
private var mBitmapaPositionHandle = 0
private var mBitmapaTextureHandle = 0
var bitmapTextureId = -12345
private set
fun drawFrame(st: SurfaceTexture, sourceWidth: Int, sourceHeight: Int, targetWidth: Int, targetHeight: Int, flipHorizontally: Boolean, flipVertically: Boolean)
if (gles == 2)
/** Draw Frame **/
println("drawFrame")
checkGlError("onDrawFrame start")
st.getTransformMatrix(mSTMatrix)
//GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f)
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f)
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT)
GLES20.glUseProgram(mProgram)
checkGlError("glUseProgram")
// original
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId)
// vertices
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET)
GLES20.glVertexAttribPointer(
maPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices
)
checkGlError("glVertexAttribPointer maPosition")
GLES20.glEnableVertexAttribArray(maPositionHandle)
checkGlError("glEnableVertexAttribArray maPositionHandle")
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET)
GLES20.glVertexAttribPointer(
maTextureHandle, 2, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices
)
checkGlError("glVertexAttribPointer maTextureHandle")
GLES20.glEnableVertexAttribArray(maTextureHandle)
checkGlError("glEnableVertexAttribArray maTextureHandle")
Matrix.setIdentityM(mMVPMatrix, 0)
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0)
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
checkGlError("glDrawArrays")
/** Draw Bitmap **/
// bitmap
// save to bitmap
val bmp = savePixels(0, 0, sourceWidth, sourceHeight)
// load texture from bitmap
println("bmp: $bmp?.width $bmp?.height")
if (bmp != null)
//val monochromeBmp = convertToBlackWhite(bmp)?.rotate(90f)
var monochromeBmp = convertToBlackWhite(bmp)
if (sourceHeight > sourceWidth)
if (monochromeBmp != null)
//monochromeBmp.rotate(90f)
println("monochromeBmp: $monochromeBmp?.width $monochromeBmp?.height")
println("drawFrame")
checkGlError("onDrawFrame start")
st.getTransformMatrix(mBitmapSTMatrix)
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f)
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT)
GLES20.glUseProgram(mBitmapProgram)
checkGlError("glUseProgram")
// original
GLES20.glActiveTexture(GLES20.GL_TEXTURE1)
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, bitmapTextureId)
// vertices
mBitmapTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET)
GLES20.glVertexAttribPointer(
mBitmapaPositionHandle, 3, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mBitmapTriangleVertices
)
checkGlError("glVertexAttribPointer mBitmapaPosition")
GLES20.glEnableVertexAttribArray(mBitmapaPositionHandle)
checkGlError("glEnableVertexAttribArray mBitmapaPositionHandle")
mBitmapTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET)
GLES20.glVertexAttribPointer(
mBitmapaTextureHandle, 2, GLES20.GL_FLOAT, false,
TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mBitmapTriangleVertices
)
checkGlError("glVertexAttribPointer mBitmapaTextureHandle")
GLES20.glEnableVertexAttribArray(mBitmapaTextureHandle)
checkGlError("glEnableVertexAttribArray maTextureHandle")
Matrix.setIdentityM(mBitmapMVPMatrix, 0)
GLES20.glUniformMatrix4fv(mBitmapuMVPMatrixHandle, 1, false, mBitmapMVPMatrix, 0)
GLES20.glUniformMatrix4fv(mBitmapuSTMatrixHandle, 1, false, mBitmapSTMatrix, 0)
if (monochromeBmp != null)
loadTexture(monochromeBmp)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
checkGlError("glDrawArrays")
// finish
GLES20.glFinish()
/**
* Save Bitmap as Grayscale
**/
private fun convertToBlackWhite(bmp: Bitmap): Bitmap?
val width = bmp.width
val height = bmp.height
val pixels = IntArray(width * height)
bmp.getPixels(pixels, 0, width, 0, 0, width, height)
val alpha = 0xFF shl 24 // ?bitmap?24?
for (i in 0 until height)
for (j in 0 until width)
var grey = pixels[width * i + j]
val red = grey and 0x00FF0000 shr 16
val green = grey and 0x0000FF00 shr 8
val blue = grey and 0x000000FF
grey = (red * 0.3 + green * 0.59 + blue * 0.11).toInt()
grey = alpha or (grey shl 16) or (grey shl 8) or grey
pixels[width * i + j] = grey
val newBmp = Bitmap.createBitmap(width, height, Bitmap.Config.RGB_565)
newBmp.setPixels(pixels, 0, width, 0, 0, width, height)
return newBmp
/**
Rotate Bitmap
*/
fun Bitmap.rotate(degrees: Float): Bitmap
val matrix = android.graphics.Matrix().apply postRotate(degrees)
return Bitmap.createBitmap(this, 0, 0, width, height, matrix, true)
/**
* Save Texture as Bitmap
**/
private fun savePixels(x: Int, y: Int, w: Int, h: Int): Bitmap?
val b = IntArray(w * (y + h))
val bt = IntArray(w * h)
val ib: IntBuffer = IntBuffer.wrap(b)
ib.position(0)
GLES20.glReadPixels(0, 0, w, h, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, ib)
var i = 0
var k = 0
while (i < h)
//remember, that OpenGL bitmap is incompatible with Android bitmap
//and so, some correction need.
for (j in 0 until w)
val pix = b[i * w + j]
val pb = pix shr 16 and 0xff
val pr = pix shl 16 and 0x00ff0000
val pix1 = pix and -0xff0100 or pr or pb
bt[(h - k - 1) * w + j] = pix1
i++
k++
return Bitmap.createBitmap(bt, w, h, Bitmap.Config.ARGB_8888)
fun saveImage(finalBitmap: Bitmap)
val generator = Random()
var n = 10000
n = generator.nextInt(n)
val fname = "Image-$n.jpg"
val file = File(output(fname)!!.path!!)
if (file.exists()) file.delete()
try
val out = FileOutputStream(file)
finalBitmap.compress(Bitmap.CompressFormat.JPEG, 90, out)
out.flush()
out.close()
catch (e: Exception)
e.printStackTrace()
private fun output(fileName: String) : Uri?
val root = mainctx!!.getExternalFilesDir(Environment.DIRECTORY_DOCUMENTS)
val myDir = File("$root/Files/")
if (!myDir.exists())
Log.d("TAG", "$myDir doesn't exist")
myDir.mkdirs()
try
val f = File(myDir, fileName)
if (f.exists())
//println("File Exists! Deleting")
f.delete()
f.createNewFile()
// file Uri
//println("fileUri: $fileUri")
return Uri.fromFile(f)
// File Saved
catch (e: FileNotFoundException)
//println("FileNotFoundException")
e.printStackTrace()
return null
catch (e: IOException)
//println("IOException")
e.printStackTrace()
return null
/**
* Load Texture from Bitmap
**/
private fun loadTexture(bitmap: Bitmap)
println("loadTexture")
// Bind to the texture in OpenGL
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, bitmapTextureId)
// Set filtering
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST)
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST)
// Load the bitmap into the bound texture.
println("bitmap: $bitmap.width $bitmap.height")
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0)
// check errors
checkGlError("texImage2d");
// Recycle the bitmap, since its data has been loaded into OpenGL.
bitmap.recycle()
/**
* Initializes GL state. Call this after the EGL surface has been created and made current.
*/
fun surfaceCreated()
println("surfaceCreated")
if (gles == 2)
// video shader
mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER)
if (mProgram == 0)
throw RuntimeException("failed creating program")
maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition")
checkGlError("glGetAttribLocation aPosition")
if (maPositionHandle == -1)
throw RuntimeException("Could not get attrib location for aPosition")
maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord")
checkGlError("glGetAttribLocation aTextureCoord")
if (maTextureHandle == -1)
throw RuntimeException("Could not get attrib location for aTextureCoord")
muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix")
checkGlError("glGetUniformLocation uMVPMatrix")
if (muMVPMatrixHandle == -1)
throw RuntimeException("Could not get attrib location for uMVPMatrix")
muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix")
checkGlError("glGetUniformLocation uSTMatrix")
if (muSTMatrixHandle == -1)
throw RuntimeException("Could not get attrib location for uSTMatrix")
val textures = IntArray(1)
GLES20.glGenTextures(1, textures, 0)
textureId = textures[0]
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId)
checkGlError("glBindTexture mTextureID")
GLES20.glTexParameterf(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST.toFloat()
)
GLES20.glTexParameterf(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR.toFloat()
)
GLES20.glTexParameteri(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE
)
GLES20.glTexParameteri(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE
)
checkGlError("glTexParameter")
// bitmap shader
mBitmapProgram = createProgram(VERTEX_SHADER, FRAGMENT_BITMAP_SHADER)
if (mBitmapProgram == 0)
throw RuntimeException("failed creating program")
mBitmapaPositionHandle = GLES20.glGetAttribLocation(mBitmapProgram, "aPosition")
checkGlError("glGetAttribLocation aPosition")
if (mBitmapaPositionHandle == -1)
throw RuntimeException("Could not get attrib location for aPosition")
mBitmapaTextureHandle = GLES20.glGetAttribLocation(mBitmapProgram, "aTextureCoord")
checkGlError("glGetAttribLocation aTextureCoord")
if (mBitmapaTextureHandle == -1)
throw RuntimeException("Could not get attrib location for aTextureCoord")
mBitmapuMVPMatrixHandle = GLES20.glGetUniformLocation(mBitmapProgram, "uMVPMatrix")
checkGlError("glGetUniformLocation uMVPMatrix")
if (mBitmapuMVPMatrixHandle == -1)
throw RuntimeException("Could not get attrib location for uMVPMatrix")
mBitmapuSTMatrixHandle = GLES20.glGetUniformLocation(mBitmapProgram, "uSTMatrix")
checkGlError("glGetUniformLocation uSTMatrix")
if (mBitmapuSTMatrixHandle == -1)
throw RuntimeException("Could not get attrib location for uSTMatrix")
val bmpTextures = IntArray(1)
GLES20.glGenTextures(1, bmpTextures, 0)
bitmapTextureId = textures[0]
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, bitmapTextureId)
checkGlError("glBindTexture bitmapTextureId")
GLES20.glTexParameterf(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST.toFloat()
)
GLES20.glTexParameterf(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR.toFloat()
)
GLES20.glTexParameteri(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE
)
GLES20.glTexParameteri(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE
)
checkGlError("glTexParameter")
/**
* Replaces the fragment shader.
*/
fun changeFragmentShader(fragmentShader: String)
if (gles == 2)
GLES20.glDeleteProgram(mProgram)
mProgram = createProgram(VERTEX_SHADER, fragmentShader)
if (mProgram == 0)
throw RuntimeException("failed creating program")
private fun loadShader(shaderType: Int, source: String): Int
if (gles == 2)
var shader = GLES20.glCreateShader(shaderType)
checkGlError("glCreateShader type=$shaderType")
GLES20.glShaderSource(shader, source)
GLES20.glCompileShader(shader)
val compiled = IntArray(1)
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0)
if (compiled[0] == 0)
Log.e(TAG, "Could not compile shader $shaderType:")
Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader))
GLES20.glDeleteShader(shader)
shader = 0
return shader
private fun createProgram(vertexSource: String, fragmentSource: String): Int
if (gles == 2)
val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource)
if (vertexShader == 0)
return 0
val pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource)
if (pixelShader == 0)
return 0
var program = GLES20.glCreateProgram()
checkGlError("glCreateProgram")
if (program == 0)
Log.e(TAG, "Could not create program")
GLES20.glAttachShader(program, vertexShader)
checkGlError("glAttachShader")
GLES20.glAttachShader(program, pixelShader)
checkGlError("glAttachShader")
GLES20.glLinkProgram(program)
val linkStatus = IntArray(1)
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0)
if (linkStatus[0] != GLES20.GL_TRUE)
Log.e(TAG, "Could not link program: ")
Log.e(TAG, GLES20.glGetProgramInfoLog(program))
GLES20.glDeleteProgram(program)
program = 0
return program
fun checkGlError(op: String)
if (gles == 2)
var error: Int
while (GLES20.glGetError().also error = it != GLES20.GL_NO_ERROR)
Log.e(TAG, "$op: glError $error")
throw RuntimeException("$op: glError $error")
companion object
private const val TAG = "TextureRender"
private const val FLOAT_SIZE_BYTES = 4
private const val TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES
private const val TRIANGLE_VERTICES_DATA_POS_OFFSET = 0
private const val TRIANGLE_VERTICES_DATA_UV_OFFSET = 3
private const val VERTEX_SHADER = "uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() \n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"\n"
private const val FRAGMENT_SHADER = "#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() \n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"\n"
private const val FRAGMENT_BITMAP_SHADER = "#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform sampler2D uTexture;\n" +
"void main() \n" +
" gl_FragColor = texture2D(uTexture, vTextureCoord);\n" +
"\n"
init
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.size * FLOAT_SIZE_BYTES
)
.order(ByteOrder.nativeOrder()).asFloatBuffer()
mTriangleVertices.put(mTriangleVerticesData).position(0)
Matrix.setIdentityM(mSTMatrix, 0)
mBitmapTriangleVertices = ByteBuffer.allocateDirect(
mBitmapTriangleVerticesData.size * FLOAT_SIZE_BYTES
)
.order(ByteOrder.nativeOrder()).asFloatBuffer()
mBitmapTriangleVertices.put(mBitmapTriangleVerticesData).position(0)
Matrix.setIdentityM(mBitmapSTMatrix, 0)
【讨论】：
以上是关于Opengl es 2.0 在视频上绘制位图叠加的主要内容，如果未能解决你的问题，请参考以下文章
OPENGL ES 2.0 知识串讲 ——OPENGL ES 详解II(传入绘制信息)