OpenCV学习3-----利用鼠标键盘回调函数实现标定人体关节点

Posted 2020-09-26 hellowOOOrld

 最近做实验，需要一些人体关节点的ground truth，需要自己手动标定，于是尝试使用OpenCV的鼠标键盘回调函数实现。

期间遇到不少问题，记录一下。

首先就是鼠标回调函数注册，

namedWindow("calibration");
setMouseCallback("calibration", onMouse,  &photo);

其中onMouse为处理鼠标事件的函数。里面需要用的一个索引selectIndex来标记当前鼠标选择的关节点是哪一个。然后当鼠标点击相应关节点并拖动时，

要时刻更新相应关节点的坐标，并更新画面。更新画面函数为：

void Public::DrawSkeleton(Mat& photo, vector<float>& x, vector<float>& y)

其中里面有一句代码特别重要就是第二行：

photo.copyTo(img);

要将当前画面拷贝到一个新的画面上，不能直接操作原来的就画面。否则画面不会更新。

还有就是在更新画面的函数里显示当前画面的imshow，不能在这个后面加上waitkey，否则就不能退出当前帧，直到栈溢出。。

imshow("calibration", img);//不能在这儿加waitkey 否则就没有退出这个函数。。栈溢出

要在主函数中使用waitkey(0),一直监听鼠标键盘的操作，写到while循环里面，否则只会更新一帧后就会卡住不动。

还有开始不知道键盘上的↑ ↓← →的ASCII码什么是什么。。于是 通过waikey返回值，将其输出到屏幕上，就这样得到了他们的键值。。。

如下是一些代码和效果图。

 

效果如图所示：

public.h头文件

#ifndef _PUBLIC_H
#define _PUBLIC_H

#include <opencv/cv.h>
#include <opencv/cvaux.h>
#include <highgui.h>
#include <fstream>
#include <string>
#include <iostream>
#include <vector>
#include <algorithm>
using namespace std;
using namespace cv;
//一些公共参数 工具函数放到此处
#define HEIGHT 480
#define WIDTH   640
#define JOINT_NUM 15
#define NUM_SUPERPIXEL 2500

class Public
{
public:
    Mat InitMat(string matrixPath, int m, int n);
    void WriteMat(Mat& m,string path);
    void DrawSkeleton(Mat& img, Mat& jointsMatrix);
    void DrawSkeleton(Mat& img, vector<float>& x, vector<float>& y);        
    vector<string> JOINTS;    
    Mat img;
    int frame;
    Public();
    ~Public(){}
    
};

#endif

public.cpp文件

#include "public.h"

Public::Public()
{
    JOINTS = { "hip", "chest", "neck", "lShldr", "lForeArm", "lHand", "rShldr", "rForeArm", "rHand", "lThigh", "lShin", "lFoot", "rThigh", "rShin", "rFoot" };
}
// 读入一个txt 返回一个m*n的矩阵
Mat Public::InitMat(string matrixPath, int m, int n)
{
    ifstream matrixFile(matrixPath);
    float temp;
    Mat mat(m, n, CV_32F);
    vector<float>xVec;//保存所有坐标
    if (matrixFile.is_open())
    {
        while (!matrixFile.eof())
        {
            matrixFile >> temp;
            xVec.push_back(temp);
        }
    }
    else
    {
        cout << "不能打开文件！" << matrixPath.c_str() << endl;
        return mat;
    }
    xVec.erase(xVec.end() - 1);
    for (int i = 0; i < m; i++)
    {
        for (int j = 0; j < n; j++)
        {
            mat.at<float>(i, j) = xVec[i * n + j];
        }
    }
    return mat;
}
//将mat矩阵 float类型 保存到path中 格式为 
void Public::WriteMat(Mat& m, string path)
{
    if (m.empty())return;
    ofstream of(path);
    for (int i = 0; i < m.rows;i++)
    {
        for (int j = 0; j < m.cols;j++)
        {
            of << m.at<float>(i, j) << " ";            
        }
        of << endl;
    }
    of.close();
}

/*
虚拟数据将关节点位置连接起来 成为一个骨架模型
0 1 hip
2 3 chest
4 5 neck
6 7 lShldr
8 9 lForeArm
10 11 lHand
12 13 rShldr
14 15 rForeArm
16 17 rHand
18 19 lThigh
20 21 lShin
22 23 lFoot
24 25 rThigh
26 27 rShin
28 29 rFoot
*/void Public::DrawSkeleton(Mat& photo, vector<float>& x, vector<float>& y)
{
    if (photo.empty())return;
    photo.copyTo(img);

    int thickness = 1;
    int lineType = 8;
    Scalar sca(0, 255, 0);
    char index[20];

    int vy = 55;
    int vx = 10;
    putText(img, "frame:"+to_string(frame), Point(vx, vy += 15), 1, 1, Scalar(0, 0, 0));
    putText(img, "s--save", Point(vx, vy += 15), 1, 1, Scalar(0, 0, 0));
    putText(img, "n--next", Point(vx, vy += 15), 1, 1, Scalar(0, 0, 0));
    putText(img, "p--previous", Point(vx, vy += 15), 1, 1, Scalar(0, 0, 0));
    
    for (int i = 0; i < JOINT_NUM; i++)
    {
        ///校验 x y坐标 可能在画面外
        x[i] = max((float)20.0, x[i]);
        x[i] = min((float)WIDTH - 5, x[i]);
        y[i] = max((float)25.0, y[i]);
        y[i] = min((float)HEIGHT - 5, y[i]);
    }
    
    for (int i = 0; i < JOINT_NUM; i++)
    {
        sprintf_s(index, "%d", i);
        circle(img, Point(x[i], y[i]), 2, Scalar(0, 255, 0), 2, 8);
        putText(img, JOINTS[i], Point(x[i], y[i]), 1, 1, Scalar(0, 0, 255));
        putText(img, index, Point(x[i] + 5, y[i] - 10), 1, 1, Scalar(0, 0, 255));

        if (i == 2 || i == 5 || i == 8 || i == 11 || i == 14)continue;

        line(img, Point(x[i], y[i]), Point(x[i + 1], y[i + 1]), sca, thickness, lineType);
    }
    line(img, Point(x[0], y[0]), Point(x[9], y[9]), sca, thickness, lineType);//hip--lthigh
    line(img, Point(x[0], y[0]), Point(x[12], y[12]), sca, thickness, lineType);//hip--rthigh
    line(img, Point(x[2], y[2]), Point(x[3], y[3]), sca, thickness, lineType);//neck--lshldr
    line(img, Point(x[2], y[2]), Point(x[6], y[6]), sca, thickness, lineType);//neck--rshldr
    imshow("calibration", img);//不能在这儿加waitkey 否则就没有退出这个函数。。栈溢出
}

 

calibration.cpp文件

/* 真实数据标定骨架点程序
1.首先没有groundtruth，只有之前产生的特征点
2.每一帧frame的特征点F乘以之前随便训练好的映射矩阵M
得到一个初步需要调整的骨架点信息S = F*M 保存起来
3.将每一个关节点编号1-15 连线 然后调整位置
4.将调整后的关节点保存 更新
*/
#include <cv.h>
#include <cvaux.h>
#include <highgui.h>
#include <fstream>
#include "ImageShow.h"
#include "Loading.h"
#include "public.h"



using namespace std;
using namespace cv;

//1首先load点云 显示出来
//2计算关节点位置 显示出来
//3调节关节点位置 保存 进入下一帧

vector<float>jointsX;
vector<float>jointsY;
Public tools;
int selectIndex = 0;

//读取关节点坐标存入 jointsX jointsY
void loadGroundTruth(string filePath)
{
    ifstream infile(filePath);
    jointsX.clear();
    jointsY.clear();
    if (infile.is_open())
    {
        float x, y;
        while (!infile.eof())
        {
            infile >> x >> y;
            jointsX.push_back(x);
            jointsY.push_back(y);
        }
    }
    else cout << "不能打开文件！" << endl;
    jointsX.pop_back();
    jointsY.pop_back();
}
//将jointsX jointsY   保存
void saveGroundTruth(string filePath)
{
    ofstream outfile(filePath);    
    for (int i = 0; i < JOINT_NUM;i++)
    {
        outfile << jointsX[i] << " "<< jointsY[i] << endl;
    }
    outfile.close();
}


void onMouse(int event, int x, int y, int flags, void* param)
{
    static bool isMouseDown = false;    
    //static int selectIndex = 0;
    //Mat *photo = (Mat*)param;
    //Mat  temp = photo->clone();
    if (event == CV_EVENT_LBUTTONDOWN)
    {         
        for (int i = 0; i < JOINT_NUM;i++)//选中某个关节
        {
            if (abs(jointsX[i] - x) < 10 && abs(jointsY[i] - y) < 10)
            {
                cout << "选中关节："<<i <<endl;
                selectIndex = i;
                isMouseDown = true;
                break;
            }
        }
    }
    if (event == CV_EVENT_LBUTTONUP)        
    {
        isMouseDown = false;        
    }    
    if (event == CV_EVENT_MOUSEMOVE)
    {
        if (isMouseDown)
        {
            jointsX[selectIndex] = x;
            jointsY[selectIndex] = y;

            tools.DrawSkeleton(*(Mat *)param, jointsX, jointsY);//更新画面        
        }
    } 
    return;

}
Mat InitMat(string matrixPath, int m, int n);
//利用已经有的特征点 乘以 映射矩阵 生成初步估计的关节点 
void generateJoints()
{
    Mat projectMat = tools.InitMat("E:/MatrixT.txt", 32, 30);    
    char  featurePath[128];
    ofstream errlog("output/errlog.txt", ios_base::app);
    for (int model = 1; model <= 6;model++)
    {
        for (int action = 1; action <= 14;action++)
        {
            for (int frame = 0; frame < 125;frame++)
            {
                sprintf_s(featurePath, "E:/laboratory/dataset/realdataresults/model%d/action%d/%d/clusterpoint.txt",model,action,frame);
                cout << featurePath << endl;
                ifstream isexist(featurePath);
                if (!isexist.is_open())//当前不存在 
                {
                    continue;
                }
                Mat featrueMat = tools.InitMat(featurePath, 1, 32);
                if (featrueMat.empty())//说明为空
                {
                    errlog << featurePath << " 不存在"<<endl;
                    cout << featurePath << " 不存在" << endl;
                    errlog.close();
                    continue;
                }
                Mat guessJoints = featrueMat * projectMat;
                char temp[128];
                sprintf_s(temp, "E:/laboratory/dataset/realdataresults/model%d/action%d/%d/guessJoints.txt", model, action, frame);
                tools.WriteMat(guessJoints, temp);
            }
        }
    }
    
}

//读入关节点位置 并标号 连线  调整位置 更新
void calibration()
{
    Mat projectMat = tools.InitMat("E:/MatrixT.txt", 32, 30);
    char  pointCloud[128];
    char joints[128];
    for (int action = 1; action <= 14; action++)
    {
        for (int model = 1; model <= 6; model++)
        {
            for (int frame = 0; frame < 125; frame++)
            {
                sprintf_s(pointCloud, "E:/laboratory/dataset/realdata/action%d/model%d/%dfinal.txt", action, model, frame);
                ifstream isexist(pointCloud);
                if (!isexist.is_open())continue;//当前不存在
                cout << pointCloud << endl;

                ImageShow ShowTool;
                Mat photo(HEIGHT, WIDTH, CV_8UC3);
                vector<Data> src;
                Loading Load;
                Load.DownLoad_Info(pointCloud, src, 1); 
                photo = ShowTool.getPhoto(src);// 加载点云
                
                
                sprintf_s(joints, "E:/laboratory/dataset/realdataresults/model%d/action%d/%d/guessJoints.txt", model, action, frame);
                ifstream isexist2(joints);
                if (!isexist2.is_open())continue;//当前不存在
                cout << joints << endl;
                loadGroundTruth(joints);//关节点
                 

                namedWindow("calibration");
                setMouseCallback("calibration", onMouse,  &photo);
                tools.frame = frame;
                tools.DrawSkeleton(photo, jointsX, jointsY);

                int keyValue = 0;
                bool processFlag = true;
                while (processFlag)
                {
                    keyValue = waitKey(0);//没有这句话会卡住不动。。。

                    switch (keyValue)
                    {
                    case \'p\':    //重新加载上一帧
                        if (frame >= 0)
                        {
                            if (frame == 0)frame = -1;                            
                            else frame -= 2;                            
                            processFlag = false;
                        }
                        break;
                    case \'s\'://save
                    case \'S\':
                        saveGroundTruth(joints);
                        cout << "success save"<< endl;
                        break;
                    case \'n\':
                    case \'N\'://next frame
                        processFlag = false;
                        break;
                    case 2424832:////left
                        
                        jointsX[selectIndex] -= 1;
                        break;

                    case 2490368://38://up
                        jointsY[selectIndex] -= 1;
                        break;

                    case 2555904://39://right                        
                        jointsX[selectIndex] += 1;
                        break;

                    case 2621440://40://down                        
                        jointsY[selectIndex] += 1;
                        break;
                    default:
                        break;
                    }

                    tools.DrawSkeleton(photo, jointsX, jointsY);
                }                
            }
        }
    }
}

int main()
{
     
    //generateJoints();

    calibration();
    
    getchar();
    return 0;
}