基于虹软人脸识别，实现RTMP直播推流追踪视频中所有人脸信息（C#）

Posted 2021-06-19 dotNET跨平台

大家应该都知道几个很常见的例子，比如在张学友的演唱会，在安检通道检票时，通过人像识别系统成功识别捉了好多在逃人员，被称为逃犯克星；人行横道不遵守交通规则闯红灯的路人被人脸识别系统抓拍放在大屏上以示警告；参加某次活动通过人脸进行签到来统计实时人流量等等， 我现在也来做一个通过电视直播，追踪画面中所有人脸信息，并捕获我需要的目标人物。

具体思路及流程

基于虹软人脸识别，对直播画面中的每一帧图片进行检测，得到图片中所有人脸信息。可以添加目标人物的照片，用目标人物的人脸特征值与直播画面帧图片中人脸信息列表中的每一个特征值进行比对。如果有匹配到目标人物，把直播画面抓拍。具体流程如下：

项目结构

播放地址我们可以在网上搜索一下电视直播RTMP地址，在程序中可进行播放

private void PlayVideo()
{
    videoCapture = new VideoCapture(rtmp);
    if (videoCapture.IsOpened())
    {
        videoInfo.Filename = rtmp;
        videoInfo.Width = (int)videoCapture.FrameWidth;
        videoInfo.Height = (int)videoCapture.FrameHeight;
        videoInfo.Fps = (int)videoCapture.Fps;

        myTimer.Interval = videoInfo.Fps == 0 ? 300 : 1000 / videoInfo.Fps;
        IsStartPlay = true;
        myTimer.Start();
    }
    else
    {
        MessageBox.Show("视频源异常");
    }
}

private void MyTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
    try
    {
        if (IsStartPlay)
        {
            lock (LockHelper)
            {
                var frame = videoCapture.RetrieveMat();
                if (frame != null)
                {
                    if (frame.Width == videoInfo.Width && frame.Height == videoInfo.Height)
                        this.SetVideoCapture(frame);
                    else
                        LogHelper.Log($"bad frame");
                }
            }
        }
    }catch(Exception ex)
    {
        LogHelper.Log(ex.Message);
    }
}
Bitmap btm = null;
private void SetVideoCapture(Mat frame)//视频捕获
{
    try
    {
        btm = OpenCvSharp.Extensions.BitmapConverter.ToBitmap(frame);
        pic_Video.Image = btm;
    }catch(Exception ex)
    {
        LogHelper.Log(ex.Message);
    }
}

以上这些就是在OpenCv中通过VideoCaptrue类对视频进行读取操作，然后把图像渲染到PictureBox控件上。在PictureBox的Paint事件中进行人脸识别与比对操作。

/// <summary>
/// 比对函数，将每一帧抓拍的照片和目标人物照片进行比对
/// </summary>
/// <param name="bitmap"></param>
/// <param name="e"></param>
/// <returns></returns>
private void CompareImgWithIDImg(Bitmap bitmap, PaintEventArgs e)
{
    if (bitmap != null)
    {
        //保证只检测一帧，防止页面卡顿以及出现其他内存被占用情况
        if (isLock == false)
        {
            isLock = true;
            Graphics g = e.Graphics;
            float offsetX = (pic_Video.Width * 1f / bitmap.Width);
            float offsetY = (pic_Video.Height * 1f / bitmap.Height);
            //根据Bitmap 获取人脸信息列表
            List<FaceInfoModel> list = FaceUtil.GetFaceInfos(pImageEngine, bitmap);
            foreach (FaceInfoModel sface in list)
            {
                //异步处理提取特征值和比对，不然页面会比较卡
                ThreadPool.QueueUserWorkItem(new WaitCallback(delegate
                {
                    try
                    {
                        //提取人脸特征
                        float similarity = CompareTwoFeatures(sface.feature, imageTemp);
                        if (similarity > threshold)
                        {
                            this.pic_cutImg.Image = bitmap;
                            this.Invoke((Action)(() =>
                            {
                                this.lbl_simiValue.Text = similarity.ToString();
                            }));
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine(ex.Message);
                    }
                }));

                MRECT rect = sface.faceRect;
                float x = rect.left * offsetX;
                float width = rect.right * offsetX - x;
                float y = rect.top * offsetY;
                float height = rect.bottom * offsetY - y;
                //根据Rect进行画框
                g.DrawRectangle(pen, x, y, width, height);
                trackUnit.message = "年龄：" + sface.age.ToString() + "\\r\\n" + "性别：" + (sface.gender == 0 ? "男" : "女");
                g.DrawString(trackUnit.message, font, brush, x, y + 5);
            }
            isLock = false;
        }
    }
}

 单张图片可能包含多张人脸，我们用FaceInfoModel 人脸信息实体类，把人脸信息放在此类中。

public class FaceInfoModel
{
    /// <summary>
    /// 年龄
    /// </summary>
    public int age { get; set; }
    /// <summary>
    /// 性别
    /// </summary>
    public int gender { get; set; }
    public ASF_Face3DAngle face3dAngle { get; set; }
    /// <summary>
    /// 人脸框
    /// </summary>
    public MRECT faceRect { get; set; }
    /// <summary>
    /// 人脸角度
    /// </summary>
    public int faceOrient { get; set; }
    /// <summary>
    /// 单人脸特征
    /// </summary>
    public IntPtr feature { get; set; }
}

多人脸实体类存放单人脸信息列表

public class MultiFaceModel : IDisposable
{
    /// <summary>
    /// 多人脸信息
    /// </summary>
    public ASF_MultiFaceInfo MultiFaceInfo { get; private set; }

    /// <summary>
    /// 单人脸信息List
    /// </summary>
    public List<ASF_SingleFaceInfo> FaceInfoList { get; private set; }

    /// <summary>
    /// 人脸信息列表
    /// </summary>
    /// <param name="multiFaceInfo"></param>
    public MultiFaceModel(ASF_MultiFaceInfo multiFaceInfo) 
    {
        this.MultiFaceInfo = multiFaceInfo;
        this.FaceInfoList = new List<ASF_SingleFaceInfo>();
        FaceInfoList = PtrToMultiFaceArray(multiFaceInfo.faceRects, multiFaceInfo.faceOrients, multiFaceInfo.faceNum);
    }
    /// <summary>
    /// 指针转多人脸列表
    /// </summary>
    /// <param name="faceRect"></param>
    /// <param name="faceOrient"></param>
    /// <param name="length"></param>
    /// <returns></returns>
    private List<ASF_SingleFaceInfo> PtrToMultiFaceArray(IntPtr faceRect, IntPtr faceOrient, int length)
    {
        List<ASF_SingleFaceInfo> FaceInfoList = new List<ASF_SingleFaceInfo>();
        var size = Marshal.SizeOf(typeof(int));
        var sizer = Marshal.SizeOf(typeof(MRECT));

        for (var i = 0; i < length; i++)
        {
            ASF_SingleFaceInfo faceInfo = new ASF_SingleFaceInfo();

            MRECT rect = new MRECT();
            var iPtr = new IntPtr(faceRect.ToInt32() + i * sizer);
            rect = (MRECT)Marshal.PtrToStructure(iPtr, typeof(MRECT));
            faceInfo.faceRect = rect;

            int orient = 0;
            iPtr = new IntPtr(faceOrient.ToInt32() + i * size);
            orient = (int)Marshal.PtrToStructure(iPtr, typeof(int));
            faceInfo.faceOrient = orient;
            FaceInfoList.Add(faceInfo);
        }
        return FaceInfoList;
    }

    public void Dispose()
    {
        Marshal.FreeCoTaskMem(MultiFaceInfo.faceRects);
        Marshal.FreeCoTaskMem(MultiFaceInfo.faceOrients);
    }
}

然后获取所有人脸信息，放在列表中备用

/// <summary>
/// 获取人脸信息列表
/// </summary>
/// <param name="pEngine"></param>
/// <param name="bitmap"></param>
/// <returns></returns>
public static List<FaceInfoModel>GetFaceInfos(IntPtr pEngine,Image bitmap)
{
    List<FaceInfoModel> listRet = new List<FaceInfoModel>();
    try
    {
        List<int> AgeList = new List<int>();
        List<int> GenderList = new List<int>();
        //检测人脸，得到Rect框
        ASF_MultiFaceInfo multiFaceInfo = FaceUtil.DetectFace(pEngine, bitmap);
        MultiFaceModel multiFaceModel = new MultiFaceModel(multiFaceInfo);
        //人脸信息处理
        ImageInfo imageInfo = ImageUtil.ReadBMP(bitmap);
        int retCode = ASFFunctions.ASFProcess(pEngine, imageInfo.width, imageInfo.height, imageInfo.format, imageInfo.imgData, ref multiFaceInfo, FaceEngineMask.ASF_AGE| FaceEngineMask.ASF_GENDER);
        //获取年龄信息
        ASF_AgeInfo ageInfo = new ASF_AgeInfo();
        retCode = ASFFunctions.ASFGetAge(pEngine, ref ageInfo);
        AgeList = ageInfo.PtrToAgeArray(ageInfo.ageArray, ageInfo.num);
        //获取性别信息
        ASF_GenderInfo genderInfo = new ASF_GenderInfo();
        retCode = ASFFunctions.ASFGetGender(pEngine, ref genderInfo);
        GenderList = genderInfo.PtrToGenderArray(genderInfo.genderArray, genderInfo.num);

        for (int i = 0; i < multiFaceInfo.faceNum; i++)
        {
            FaceInfoModel faceInfo = new FaceInfoModel();
            faceInfo.age = AgeList[i];
            faceInfo.gender = GenderList[i];
            faceInfo.faceRect = multiFaceModel.FaceInfoList[i].faceRect;
            faceInfo.feature = ExtractFeature(pEngine, bitmap, multiFaceModel.FaceInfoList[i]);//提取单人脸特征
            faceInfo.faceOrient = multiFaceModel.FaceInfoList[i].faceOrient;
            listRet.Add(faceInfo);
        }
        return listRet;//返回多人脸信息
    }
    catch {
        return listRet;
    }
}

从列表中获取到的多张人脸，在人脸上画框作出标识，也可以把提取的人脸信息，年龄、性别作出展示。接下来就是选择一张目标人物的照片，通过SDK提取目标人物的人脸特征值作为比较对象，逐一与视频中的人脸特征进行比较。如果有判断到相似度匹配的人脸，则把视频帧图像呈现出来。

/// <summary>
/// 比较两个特征值的相似度,返回相似度
/// </summary>
/// <param name="feature1"></param>
/// <param name="feature2"></param>
/// <returns></returns>
private float CompareTwoFeatures(IntPtr feature1, IntPtr feature2)
{
    float similarity = 0.0f;
    //调用人脸匹配方法，进行匹配
    ASFFunctions.ASFFaceFeatureCompare(pImageEngine, feature1, feature2, ref similarity);
    return similarity;
}

之前只实现了从多张人脸中获取一张最大尺寸的人脸作为比较对象，这样视频中也就只能对一张人脸进行画框标记了，现在是把所有提取到的人脸均进行标记，并把各自特征值存在列表中，以便与目标人脸特征值进行匹配。

这样也就粗略的实现了人脸识别追踪，并对目标人物进行抓拍的功能了。

GitHub源码已上传:https://github.com/yumaster/FaceTracking