EasyPR--开发详解文字定位
Posted 计算机的潜意识
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了EasyPR--开发详解文字定位相关的知识，希望对你有一定的参考价值。
今天我们来介绍车牌定位中的一种新方法--文字定位方法（MSER），包括其主要设计思想与实现。接着我们会介绍一下EasyPR v1.5-beta版本中带来的几项改动。
一. 文字定位法
在EasyPR前面几个版本中，最为人所诟病的就是定位效果不佳，尤其是在面对生活场景（例如手机拍摄）时。由于EasyPR最早的数据来源于卡口，因此对卡口数据进行了优化，而并没有对生活场景中图片有较好处理的策略。后来一个版本（v1.3）增加了颜色定位方法，改善了这种现象，但是对分辨率较大的图片处理仍然不好。再加上颜色定位在面对低光照，低对比度的图像时处理效果大幅度下降，颜色本身也是一个不稳定的特征。因此EasyPR的车牌定位的整体鲁棒性仍然不足。
针对这种现象，EasyPR v1.5增加了一种新的定位方法，文字定位方法，大幅度改善了这些问题。下面几幅图可以说明文字定位法的效果。
图1 夜间的车牌图像（左） ， 图2 对比度非常低的图像（右）
图3 近距离的图像（左） ， 图4 高分辨率的图像（右）
图1是夜间的车牌图像，图2是对比度非常低的图像，图3是非常近距离拍摄的图像，图4则是高分辨率（3200宽）的图像。
文字定位方法是采用了低级过滤器提取文字，然后再将其组合的一种定位方法。原先是利用在场景中定位文字，在这里利用其定位车牌。与在扫描文档中的文字不同，自然场景中的文字具有低对比度，背景各异，光亮干扰较多等情况，因此需要一个极为鲁棒的方法去提取出来。目前业界用的较多的是MSER（最大稳定极值区域）方法。EasyPR使用的是MSER的一个改良方法，专门针对文字进行了优化。在文字定位出来以后，一般需要用一个分类器将其中大部分的定位错误的文字去掉，例如ANN模型。为了获得最终的车牌，这些文字需要组合起来。由于实际情况的复杂，简单的使用普通的聚类效果往往不好，因此EasyPR使用了一种鲁棒性较强的种子生长方法（seed growing）去组合。
我在这里简单介绍一下具体的实现。关于方法的细节可以看代码，有很多的注释（代码可能较长）。关于方法的思想可以看附录的两篇论文。
1 //! use verify size to first generate char candidates 2 void mserCharMatch(const Mat &src, std::vector<Mat> &match, std::vector<CPlate>& out_plateVec_blue, std::vector<CPlate>& out_plateVec_yellow, 3 bool usePlateMser, std::vector<RotatedRect>& out_plateRRect_blue, std::vector<RotatedRect>& out_plateRRect_yellow, int img_index, 4 bool showDebug) { 5 Mat image = src; 6 7 std::vector<std::vector<std::vector<Point>>> all_contours; 8 std::vector<std::vector<Rect>> all_boxes; 9 all_contours.resize(2); 10 all_contours.at(0).reserve(1024); 11 all_contours.at(1).reserve(1024); 12 all_boxes.resize(2); 13 all_boxes.at(0).reserve(1024); 14 all_boxes.at(1).reserve(1024); 15 16 match.resize(2); 17 18 std::vector<Color> flags; 19 flags.push_back(BLUE); 20 flags.push_back(YELLOW); 21 22 const int imageArea = image.rows * image.cols; 23 const int delta = 1; 24 //const int delta = CParams::instance()->getParam2i();; 25 const int minArea = 30; 26 const double maxAreaRatio = 0.05; 27 28 Ptr<MSER2> mser; 29 mser = MSER2::create(delta, minArea, int(maxAreaRatio * imageArea)); 30 mser->detectRegions(image, all_contours.at(0), all_boxes.at(0), all_contours.at(1), all_boxes.at(1)); 31 32 // mser detect 33 // color_index = 0 : mser-, detect white characters, which is in blue plate. 34 // color_index = 1 : mser+, detect dark characters, which is in yellow plate. 35 36 #pragma omp parallel for 37 for (int color_index = 0; color_index < 2; color_index++) { 38 Color the_color = flags.at(color_index); 39 40 std::vector<CCharacter> charVec; 41 charVec.reserve(128); 42 43 match.at(color_index) = Mat::zeros(image.rows, image.cols, image.type()); 44 45 Mat result = image.clone(); 46 cvtColor(result, result, COLOR_GRAY2BGR); 47 48 size_t size = all_contours.at(color_index).size(); 49 50 int char_index = 0; 51 int char_size = 20; 52 53 // Chinese plate has max 7 characters. 54 const int char_max_count = 7; 55 56 // verify char size and output to rects; 57 for (size_t index = 0; index < size; index++) { 58 Rect rect = all_boxes.at(color_index)[index]; 59 std::vector<Point>& contour = all_contours.at(color_index)[index]; 60 61 // sometimes a plate could be a mser rect, so we could 62 // also use mser algorithm to find plate 63 if (usePlateMser) { 64 RotatedRect rrect = minAreaRect(Mat(contour)); 65 if (verifyRotatedPlateSizes(rrect)) { 66 //rotatedRectangle(result, rrect, Scalar(255, 0, 0), 2); 67 if (the_color == BLUE) out_plateRRect_blue.push_back(rrect); 68 if (the_color == YELLOW) out_plateRRect_yellow.push_back(rrect); 69 } 70 } 71 72 // find character 73 if (verifyCharSizes(rect)) { 74 Mat mserMat = adaptive_image_from_points(contour, rect, Size(char_size, char_size)); 75 Mat charInput = preprocessChar(mserMat, char_size); 76 Rect charRect = rect; 77 78 Point center(charRect.tl().x + charRect.width / 2, charRect.tl().y + charRect.height / 2); 79 Mat tmpMat; 80 double ostu_level = cv::threshold(image(charRect), tmpMat, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); 81 82 //cv::circle(result, center, 3, Scalar(0, 0, 255), 2); 83 84 // use judegMDOratio2 function to 85 // remove the small lines in character like "zh-cuan" 86 if (judegMDOratio2(image, rect, contour, result)) { 87 CCharacter charCandidate; 88 charCandidate.setCharacterPos(charRect); 89 charCandidate.setCharacterMat(charInput); 90 charCandidate.setOstuLevel(ostu_level); 91 charCandidate.setCenterPoint(center); 92 charCandidate.setIsChinese(false); 93 charVec.push_back(charCandidate); 94 } 95 } 96 } 97 98 // improtant, use matrix multiplication to acclerate the 99 // classification of many samples. use the character 100 // score, we can use non-maximum superssion (nms) to 101 // reduce the characters which are not likely to be true 102 // charaters, and use the score to select the strong seed 103 // of which the score is larger than 0.9 104 CharsIdentify::instance()->classify(charVec); 105 106 // use nms to remove the character are not likely to be true. 107 double overlapThresh = 0.6; 108 //double overlapThresh = CParams::instance()->getParam1f(); 109 NMStoCharacter(charVec, overlapThresh); 110 charVec.shrink_to_fit(); 111 112 std::vector<CCharacter> strongSeedVec; 113 strongSeedVec.reserve(64); 114 std::vector<CCharacter> weakSeedVec; 115 weakSeedVec.reserve(64); 116 std::vector<CCharacter> littleSeedVec; 117 littleSeedVec.reserve(64); 118 119 //size_t charCan_size = charVec.size(); 120 for (auto charCandidate : charVec) { 121 //CCharacter& charCandidate = charVec[char_index]; 122 Rect rect = charCandidate.getCharacterPos(); 123 double score = charCandidate.getCharacterScore(); 124 if (charCandidate.getIsStrong()) { 125 strongSeedVec.push_back(charCandidate); 126 } 127 else if (charCandidate.getIsWeak()) { 128 weakSeedVec.push_back(charCandidate); 129 //cv::rectangle(result, rect, Scalar(255, 0, 255)); 130 } 131 else if (charCandidate.getIsLittle()) { 132 littleSeedVec.push_back(charCandidate); 133 //cv::rectangle(result, rect, Scalar(255, 0, 255)); 134 } 135 } 136 137 std::vector<CCharacter> searchCandidate = charVec; 138 139 // nms to srong seed, only leave the strongest one 140 overlapThresh = 0.3; 141 NMStoCharacter(strongSeedVec, overlapThresh); 142 143 // merge chars to group 144 std::vector<std::vector<CCharacter>> charGroupVec; 145 charGroupVec.reserve(64); 146 mergeCharToGroup(strongSeedVec, charGroupVec); 147 148 // genenrate the line of the group 149 // based on the assumptions , the mser rects which are 150 // given high socre by character classifier could be no doubtly 151 // be the characters in one plate, and we can use these characeters 152 // to fit a line which is the middle line of the plate. 153 std::vector<CPlate> plateVec; 154 plateVec.reserve(16); 155 for (auto charGroup : charGroupVec) { 156 Rect plateResult = charGroup[0].getCharacterPos(); 157 std::vector<Point> points; 158 points.reserve(32); 159 160 Vec4f line; 161 int maxarea = 0; 162 Rect maxrect; 163 double ostu_level_sum = 0; 164 165 int leftx = image.cols; 166 Point leftPoint(leftx, 0); 167 int rightx = 0; 168 Point rightPoint(rightx, 0); 169 170 std::vector<CCharacter> mserCharVec; 171 mserCharVec.reserve(32); 172 173 // remove outlier CharGroup 174 std::vector<CCharacter> roCharGroup; 175 roCharGroup.reserve(32); 176 177 removeRightOutliers(charGroup, roCharGroup, 0.2, 0.5, result); 178 //roCharGroup = charGroup; 179 180 for (auto character : roCharGroup) { 181 Rect charRect = character.getCharacterPos(); 182 cv::rectangle(result, charRect, Scalar(0, 255, 0), 1); 183 plateResult |= charRect; 184 185 Point center(charRect.tl().x + charRect.width / 2, charRect.tl().y + charRect.height / 2); 186 points.push_back(center); 187 mserCharVec.push_back(character); 188 //cv::circle(result, center, 3, Scalar(0, 255, 0), 2); 189 190 ostu_level_sum += character.getOstuLevel(); 191 192 if (charRect.area() > maxarea) { 193 maxrect = charRect; 194 maxarea = charRect.area(); 195 } 196 if (center.x < leftPoint.x) { 197 leftPoint = center; 198 } 199 if (center.x > rightPoint.x) { 200 rightPoint = center; 201 } 202 } 203 204 double ostu_level_avg = ostu_level_sum / (double)roCharGroup.size(); 205 if (1 && showDebug) { 206 std::cout << "ostu_level_avg:" << ostu_level_avg << std::endl; 207 } 208 float ratio_maxrect = (float)maxrect.width / (float)maxrect.height; 209 210 if (points.size() >= 2 && ratio_maxrect >= 0.3) { 211 fitLine(Mat(points), line, CV_DIST_L2, 0, 0.01, 0.01); 212 213 float k = line[1] / line[0]; 214 //float angle = atan(k) * 180 / (float)CV_PI; 215 //std::cout << "k:" << k << std::endl; 216 //std::cout << "angle:" << angle << std::endl; 217 //std::cout << "cos:" << 0.3 * cos(k) << std::endl; 218 //std::cout << "ratio_maxrect:" << ratio_maxrect << std::endl; 219 220 std::sort(mserCharVec.begin(), mserCharVec.end(), 221 [](const CCharacter& r1, const CCharacter& r2) { 222 return r1.getCharacterPos().tl().x < r2.getCharacterPos().tl().x; 223 }); 224 225 CCharacter midChar = mserCharVec.at(int(mserCharVec.size() / 2.f)); 226 Rect midRect = midChar.getCharacterPos(); 227 Point midCenter(midRect.tl().x + midRect.width / 2, midRect.tl().y + midRect.height / 2); 228 229 int mindist = 7 * maxrect.width; 230 std::vector<Vec2i> distVecVec; 231 distVecVec.reserve(32); 232 233 Vec2i mindistVec; 234 Vec2i avgdistVec; 235 236 // computer the dist which is the distacne between 237 // two near characters in the plate, use dist we can 238 // judege how to computer the max search range, and choose the 239 // best location of the sliding window in the next steps. 240 for (size_t mser_i = 0; mser_i + 1 < mserCharVec.size(); mser_i++) { 241 Rect charRect = mserCharVec.at(mser_i).getCharacterPos(); 242 Point center(charRect.tl().x + charRect.width / 2, charRect.tl().y + charRect.height / 2); 243 244 Rect charRectCompare = mserCharVec.at(mser_i + 1).getCharacterPos(); 245 Point centerCompare(charRectCompare.tl().x + charRectCompare.width / 2, 246 charRectCompare.tl().y + charRectCompare.height / 2); 247 248 int dist = charRectCompare.x - charRect.x; 249 Vec2i distVec(charRectCompare.x - charRect.x, charRectCompare.y - charRect.y); 250 distVecVec.push_back(distVec); 251 252 //if (dist < mindist) { 253 // mindist = dist; 254 // mindistVec = distVec; 255 //} 256 } 257 258 std::sort(distVecVec.begin(), distVecVec.end(), 259 [](const Vec2i& r1, const Vec2i& r2) { 260 return r1[0] < r2[0]; 261 }); 262 263 avgdistVec = distVecVec.at(int((distVecVec.size() - 1) / 2.f)); 264 265 //float step = 10.f * (float)maxrect.width; 266 //float step = (float)mindistVec[0]; 267 float step = (float)avgdistVec[0]; 268 269 //cv::line(result, Point2f(line[2] - step, line[3] - k*step), Point2f(line[2] + step, k*step + line[3]), Scalar(255, 255, 255)); 270 cv::line(result, Point2f(midCenter.x - step, midCenter.y - k*step), Point2f(midCenter.x + step, k*step + midCenter.y), Scalar(255, 255, 255)); 271 //cv::circle(result, leftPoint, 3, Scalar(0, 0, 255), 2); 272 273 CPlate plate; 274 plate.setPlateLeftPoint(leftPoint); 275 plate.setPlateRightPoint(rightPoint); 276 277 plate.setPlateLine(line); 278 plate.setPlatDistVec(avgdistVec); 279 plate.setOstuLevel(ostu_level_avg); 280 281 plate.setPlateMergeCharRect(plateResult); 282 plate.setPlateMaxCharRect(maxrect); 283 plate.setMserCharacter(mserCharVec); 284 plateVec.push_back(plate); 285 } 286 } 287 288 // use strong seed to construct the first shape of the plate, 289 // then we need to find characters which are the weak seed. 290 // because we use strong seed to build the middle lines of the plate, 291 // we can simply use this to consider weak seeds only lie in the 292 // near place of the middle line 293 for (auto plate : plateVec) { 294 Vec4f line = plate.getPlateLine(); 295 Point leftPoint = plate.getPlateLeftPoint(); 296 Point rightPoint = plate.getPlateRightPoint(); 297 298 Rect plateResult = plate.getPlateMergeCharRect(); 299 Rect maxrect = plate.getPlateMaxCharRect(); 300 Vec2i dist = plate.getPlateDistVec(); 301 double ostu_level = plate.getOstuLevel(); 302 303 std::vector<CCharacter> mserCharacter = plate.getCopyOfMserCharacters(); 304 mserCharacter.reserve(16); 305 306 float k = line[1] / line[0]; 307 float x_1 = line[2]; 308 float y_1 = line[3]; 309 310 std::vector<CCharacter> searchWeakSeedVec; 311 searchWeakSeedVec.reserve(16); 312 313 std::vector<CCharacter> searchRightWeakSeed; 314 searchRightWeakSeed.reserve(8); 315 std::vector<CCharacter> searchLeftWeakSeed; 316 searchLeftWeakSeed.reserve(8); 317 318 std::vector<CCharacter> slideRightWindow; 319 slideRightWindow.reserve(8); 320 std::vector<CCharacter> slideLeftWindow; 321 slideLeftWindow.reserve(8); 322 323 // draw weak seed and little seed from line; 324 // search for mser rect 325 if (1 && showDebug) { 326 std::cout << "search for mser rect:" << std::endl; 327 } 328 329 if (0 && showDebug) { 330 std::stringstream ss(std::stringstream::in | std::stringstream::out); 331 ss << "resources/image/tmp/" << img_index << "_1_" << "searcgMserRect.jpg"; 332 imwrite(ss.str(), result); 333 } 334 if (1 && showDebug) { 335 std::cout << "mserCharacter:" << mserCharacter.size() << std::endl; 336 } 337 338 // if the count of strong seed is larger than max count, we dont need 339 // all the next steps, if not, we first need to search the weak seed in 340 // the same line as the strong seed. The judge condition contains the distance 341 // between strong seed and weak seed , and the rect simily of each other to improve 342 // the roubustnedd of the seed growing algorithm. 343 if (mserCharacter.size() < char_max_count) { 344 double thresh1 = 0.15; 345 double thresh2 = 2.0; 346 searchWeakSeed(searchCandidate, searchRightWeakSeed, thresh1, thresh2, line, rightPoint, 347 maxrect, plateResult, result, CharSearchDirection::RIGHT); 348 if (1 && showDebug) { 349 std::cout << "searchRightWeakSeed:" << searchRightWeakSeed.size() << std::endl; 350 } 351 for (auto seed : searchRightWeakSeed) { 352 cv::rectangle(result, seed.getCharacterPos(), Scalar(255, 0, 0), 1); 353 mserCharacter.push_back(seed); 354 } 355 356 searchWeakSeed(searchCandidate, searchLeftWeakSeed, thresh1, thresh2, line, leftPoint, 357 maxrect, plateResult, result, CharSearchDirection::LEFT); 358 if (1 && showDebug) { 359 std::cout << "searchLeftWeakSeed:" << searchLeftWeakSeed.size() << std::endl; 360 } 361 for (auto seed : searchLeftWeakSeed) { 362 cv::rectangle(result, seed.getCharacterPos(), Scalar(255, 0, 0), 1); 363 mserCharacter.push_back(seed); 364 } 365 } 366 367 // sometimes the weak seed is in the middle of the strong seed. 368 // and sometimes two strong seed are actually the two parts of one character. 369 //EasyPR--开发详解颜色定位与偏斜扭转