opencv 视觉项目学习笔记（二）： 基于 svm 和 knn 车牌识别

车牌识别的属于常见的 模式识别 ，其基本流程为下面三个步骤：

1） 分割： 检测并检测图像中感兴趣区域；

2）特征提取： 对字符图像集中的每个部分进行提取；

3）分类： 判断图像快是不是车牌或者 每个车牌字符的分类。

车牌识别分为两个步骤， 车牌检测， 车牌识别， 都属于模式识别。

基本结构如下：

一、车牌检测

　　1、车牌局部化（分割车牌区域），根据尺寸等基本信息去除非车牌图像；

　　2、判断车牌是否存在 （训练支持向量机 -svm， 判断车牌是否存在）。

二、车牌识别

　　1、字符局部化（分割字符），根据尺寸等信息剔除不合格图像

　　2、字符识别 （ knn  分类）

1.1 车牌局部化、并剔除不合格区域　　

vector<Plate> DetectRegions::segment(Mat input) {
    vector<Plate> output;

    //转为灰度图，并去噪
    Mat img_gray;
    cvtColor(input, img_gray, CV_BGR2GRAY);
    blur(img_gray, img_gray, Size(5, 5));

    //找垂直边
    Mat img_sobel;
    Sobel(img_gray, img_sobel, CV_8U, 1, 0, 3, 1, 0, BORDER_DEFAULT);

    // 阈值化过滤像素
    Mat img_threshold;
    threshold(img_sobel, img_threshold, 0, 255, CV_THRESH_OTSU + CV_THRESH_BINARY);

    // 开运算
    Mat element = getStructuringElement(MORPH_RECT, Size(17, 3));
    morphologyEx(img_threshold, img_threshold, CV_MOP_CLOSE, element);

    //查找轮廓
    vector<vector<Point>> contours;
    findContours(img_threshold, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);

    vector<vector<Point>>::iterator itc = contours.begin();
    vector<RotatedRect> rects;

    // 去除面积以及宽高比不合适区域
    while (itc != contours.end())
    {
        // create bounding rect of object
        RotatedRect mr = minAreaRect(Mat(*itc));
        if (!verifySizes(mr))
        {
            itc = contours.erase(itc); 
        }
        else
        {
            ++itc;
            rects.push_back(mr);
        }
    }


    // 绘出获取区域
    cv::Mat result;
    input.copyTo(result);
    cv::drawContours(result, contours, -1, cv::Scalar(255, 0, 0), 1);

    for (int i = 0; i < rects.size(); i++) {

        //For better rect cropping for each posible box
        //Make floodfill algorithm because the plate has white background
        //And then we can retrieve more clearly the contour box
        circle(result, rects[i].center, 3, Scalar(0, 255, 0), -1);
        //get the min size between width and height
        float minSize = (rects[i].size.width < rects[i].size.height) ? rects[i].size.width : rects[i].size.height;
        minSize = minSize - minSize * 0.5;
        //initialize rand and get 5 points around center for floodfill algorithm
        srand(time(NULL));
        //Initialize floodfill parameters and variables
        Mat mask;
        mask.create(input.rows + 2, input.cols + 2, CV_8UC1);
        mask = Scalar::all(0);
        int loDiff = 30;
        int upDiff = 30;
        int connectivity = 4;
        int newMaskVal = 255;
        int NumSeeds = 10;
        Rect ccomp;
        int flags = connectivity + (newMaskVal << 8) + CV_FLOODFILL_FIXED_RANGE + CV_FLOODFILL_MASK_ONLY;
        for (int j = 0; j < NumSeeds; j++) {
            Point seed;
            seed.x = rects[i].center.x + rand() % (int)minSize - (minSize / 2);
            seed.y = rects[i].center.y + rand() % (int)minSize - (minSize / 2);
            circle(result, seed, 1, Scalar(0, 255, 255), -1);
            int area = floodFill(input, mask, seed, Scalar(255, 0, 0), &ccomp, Scalar(loDiff, loDiff, loDiff), Scalar(upDiff, upDiff, upDiff), flags);
        }
        if (showSteps)
            imshow("MASK", mask);
        //cvWaitKey(0);

        //Check new floodfill mask match for a correct patch.
        //Get all points detected for get Minimal rotated Rect
        vector<Point> pointsInterest;
        Mat_<uchar>::iterator itMask = mask.begin<uchar>();
        Mat_<uchar>::iterator end = mask.end<uchar>();
        for (; itMask != end; ++itMask)
            if (*itMask == 255)
                pointsInterest.push_back(itMask.pos());

        RotatedRect minRect = minAreaRect(pointsInterest);

        if (verifySizes(minRect)) {
            // rotated rectangle drawing 
            Point2f rect_points[4]; 
            minRect.points(rect_points);
            for (int j = 0; j < 4; j++)
                line(result, rect_points[j], rect_points[(j + 1) % 4], Scalar(0, 0, 255), 1, 8);

            // 获取旋转矩阵
            float r = (float)minRect.size.width / (float)minRect.size.height;
            float angle = minRect.angle;
            if (r < 1)
                angle = 90 + angle;
            Mat rotmat = getRotationMatrix2D(minRect.center, angle, 1);

            // 获取映射图像
            Mat img_rotated;
            warpAffine(input, img_rotated, rotmat, input.size(), CV_INTER_CUBIC);

            // Crop image
            Size rect_size = minRect.size;
            if (r < 1)
                swap(rect_size.width, rect_size.height);
            Mat img_crop;
            getRectSubPix(img_rotated, rect_size, minRect.center, img_crop);

            Mat resultResized;
            resultResized.create(33, 144, CV_8UC3);
            resize(img_crop, resultResized, resultResized.size(), 0, 0, INTER_CUBIC);
            // 直方图
            Mat grayResult;
            cvtColor(resultResized, grayResult, CV_BGR2GRAY);
            blur(grayResult, grayResult, Size(3, 3));
            grayResult = histeq(grayResult);
            output.push_back(Plate(grayResult, minRect.boundingRect()));
        }
    }

    return output;
}

 1.2 判断车牌是否存在

　　1.2.1  训练 svm

　　　　svm 会创建一个或多个超平面， 这些超级平面能判断数据属于那个类。

　　　　训练数据： 所有训练数据存储再一个 N x M 的矩阵中， 其中 N 为样本数， M 为特征数（每个样本是该训练矩阵中的一行）。这些数据  所有数据存在  xml 文件中， 

　　　　标签数据：  每个样本的类别信息存储在另一个 N x 1 的矩阵中， 每行为一个样本标签。

　　　　训练数据存放在本地 svm.xml 文件中。 

　　　　

　　　　// TrainSvm.cpp 文件

　　　　

#include <iostream>
#include <opencv2/opencv.hpp>

#include "Preprocess.h"

using namespace std;
using namespace cv;
using namespace cv::ml;

int main(int argc, char** argv)
{
    FileStorage fs;
    fs.open("SVM.xml", FileStorage::READ);
    Mat SVM_TrainingData;
    Mat SVM_Classes;
    fs["TrainingData"] >> SVM_TrainingData;
    fs["classes"] >> SVM_Classes;
    // Set SVM storage
    Ptr<ml::SVM> model = ml::SVM::create();
    model->setType(SVM::C_SVC);
    model->setKernel(SVM::LINEAR); // 核函数
    // 训练数据
    Ptr<TrainData> tData = TrainData::create(SVM_TrainingData, ROW_SAMPLE, SVM_Classes);
    // 训练分类器
    model->train(tData);
    model->save("model.xml");

    // TODO: 测试
    return 0;

　　　　// Preprocess.cpp　　　　

　　　　

#include <string>
#include <vector>
#include <fstream>
#include <algorithm>

#include "Preprocess.h"

using namespace cv;


void Preprocess::getAllFiles(string path, vector<string> &files, string fileType)
{
    long hFile = 0;
    struct _finddata_t  fileInfo;
    string p;
    if ((hFile = _findfirst(p.assign(path).append("\*" + fileType).c_str(), &fileInfo)) != -1)
    {
        do
        {
            files.push_back(p.assign(path).append("\").append(fileInfo.name));
        } while (_findnext(hFile, &fileInfo) == 0);
        _findclose(hFile);  // 关闭句柄
    }

}

void Preprocess::extract_img_data(string path_plates, string path_noPlates)
{
    cout << "OpenCV Training SVM Automatic Number Plate Recognition
";

    int imgWidth = 144;
    int imgHeight = 33;
    int numPlates = 100;
    int numNoPlates = 100;
    Mat classes;
    Mat trainingData;

    Mat trainingImages;
    vector<int> trainingLabels;

    for (int i = 0; i < numPlates; i++)
    {
        stringstream ss(stringstream::in | stringstream::out);
        ss << path_plates << i << ".jpg";
        Mat img = imread(ss.str(), 0);
        resize(img, img, Size(imgWidth, imgWidth));
        img = img.reshape(1, 1);
        trainingImages.push_back(img);
        trainingLabels.push_back(1);
    }

    for (int i = 0; i < numNoPlates; i++)
    {
        stringstream ss;
        ss << path_noPlates << i << ".jpg";
        Mat img = imread(ss.str(), 0);
        img = img.reshape(1, 1);
        trainingImages.push_back(img);
        trainingLabels.push_back(0);
    }

    Mat(trainingImages).copyTo(trainingData);
    trainingData.convertTo(trainingData, CV_32FC1);
    Mat(trainingLabels).copyTo(classes);

    FileStorage fs("SVM.xml", FileStorage::WRITE);
    fs << "TrainingData" << trainingData;
    fs << "classess" << classes;
    fs.release();
}

　　1.2.2  利用 svm 判断车牌是否存在

　　

// load model
Ptr<ml::SVM> model = SVM::load("model.xml");

// For each possible plate, classify with svm if it's plate
vector<Plate> plates;
for (int i = 0; i < posible_regions.size(); i++)
{
    Mat img = posible_regions[i].plateImg;
    Mat p = img.reshape(1, 1);
    p.convertTo(p, CV_32FC1);
    int reponse = (int)model->predict(p);
    if (reponse)
    {
        plates.push_back(posible_regions[i]);
        //bool res = imwrite("test.jpg", img);
    }
}

以上，已经找了存在车牌的区域，并保存到一个 vector 中。 

下面使用 k 邻近算法， 来识别车牌图像中的车牌字符。

2.1 字符分割

　　分割字符，并剔除不合格图像

vector<CharSegment> OCR::segment(Plate plate) {
    Mat input = plate.plateImg;
    vector<CharSegment> output;
    //使字符为白色，背景为黑色
    Mat img_threshold;
    threshold(input, img_threshold, 60, 255, CV_THRESH_BINARY_INV);

    Mat img_contours;
    img_threshold.copyTo(img_contours);
    // 找到所有物体
    vector< vector< Point> > contours;
    findContours(img_contours,
        contours, // a vector of contours
        CV_RETR_EXTERNAL, // retrieve the external contours
        CV_CHAIN_APPROX_NONE); // all pixels of each contours

    // Draw blue contours on a white image
    cv::Mat result;
    img_threshold.copyTo(result);
    cvtColor(result, result, CV_GRAY2RGB);
    cv::drawContours(result, contours,
        -1, // draw all contours
        cv::Scalar(255, 0, 0), // in blue
        1); // with a thickness of 1

    //Remove patch that are no inside limits of aspect ratio and area.    
    vector<vector<Point> >::iterator itc = contours.begin();
    while (itc != contours.end()) {

        //Create bounding rect of object
        Rect mr = boundingRect(Mat(*itc));
        rectangle(result, mr, Scalar(0, 255, 0));
        //提取合格图像区域
        Mat auxRoi(img_threshold, mr);
        if (verifySizes(auxRoi)) {
            auxRoi = preprocessChar(auxRoi);
            output.push_back(CharSegment(auxRoi, mr));
            rectangle(result, mr, Scalar(0, 125, 255));
        }
        ++itc;
    }

    return output;
}

Mat OCR::preprocessChar(Mat in) {
    //Remap image
    int h = in.rows;
    int w = in.cols;
    Mat transformMat = Mat::eye(2, 3, CV_32F);
    int m = max(w, h);
    transformMat.at<float>(0, 2) = m / 2 - w / 2;
    transformMat.at<float>(1, 2) = m / 2 - h / 2;
    // 仿射变换，将图像投射到尺寸更大的图像上（使用偏移）
    Mat warpImage(m, m, in.type());
    warpAffine(in, warpImage, transformMat, warpImage.size(), INTER_LINEAR, BORDER_CONSTANT, Scalar(0));
    Mat out;
    resize(warpImage, out, Size(charSize, charSize));

    return out;
}

2.2 字符识别

　　2.2.1 训练 knn

　　　　使用 opencv  自带的 digits.png 文件， 可以训练训练识别识别数字的 knn 。

　　　　

#include <iostream>
#include <opencv2/opencv.hpp>


using namespace cv;
using namespace std;
using namespace cv::ml;

const int numFilesChars[] = { 35, 40, 42, 41, 42, 33, 30, 31, 49, 44, 30, 24, 21, 20, 34, 9, 10, 3, 11, 3, 15, 4, 9, 12, 10, 21, 18, 8, 15, 7 };

int main()
{

    std::cout << "OpenCV Training OCR Automatic Number Plate Recognition
";

    string path = "D:/Program Files (x86)/opencv_3.4.3/opencv/sources/samples/data/digits.png";
    Mat img = imread(path);
    Mat gray;
    cvtColor(img, gray, CV_BGR2GRAY);
    int b = 20;
    int m = gray.rows / b;  // 将原图裁剪为 20 * 20 的小图块
    int n = gray.cols / b;  // 将原图裁剪为 20 * 20 的小图块

    Mat data, labels; // 特征矩阵

    // 按照列来读取数据， 每 5 个数据为一个类
    for (int i = 0; i < n; i++)
    {
        int offsetCol = i * b; // 列上的偏移量
        for (int  j = 0; j < m; j++)
        {
            int offsetRow = j * b; // 行上的偏移量
            Mat tmp;
            gray(Range(offsetRow, offsetRow + b), Range(offsetCol, offsetCol + b)).copyTo(tmp);
            data.push_back(tmp.reshape(0, 1)); // 序列化后放入特征矩阵
            labels.push_back((int)j / 5);  // 对应的标注
        }
    }
    data.convertTo(data, CV_32F);
    int samplesNum = data.rows;
    int trainNum = 3000;
    Mat trainData, trainLabels;
    trainData = data(Range(0, trainNum), Range::all()); // 前 3000 个为训练数据
    trainLabels = labels(Range(0, trainNum), Range::all()); 

    // 使用k 邻近算法那（knn， k-nearest_neighbor） 算法
    int K = 5;
    Ptr<cv::ml::TrainData> tData = cv::ml::TrainData::create(trainData, ROW_SAMPLE, trainLabels);
    Ptr<KNearest> model = KNearest::create();

    model->setDefaultK(K);        // 设定查找时返回数量为 5
    // 设置分类器为分类 或回归 
    // 分类问题：输出离散型变量（如 -1，1， 100）, 为定性输出（如预测明天是下雨、天晴还是多云）
    // 回归问题: 回归问题的输出为连续型变量，为定量输出（如明天温度为多少度）
    model->setIsClassifier(true); 
    model->train(tData);

    // 预测分类
    double train_hr = 0, test_hr = 0;
    Mat response;
    // compute prediction error on train and test data
    for (int  i = 0; i < samplesNum; i++)
    {
        Mat smaple = data.row(i);
        float r = model->predict(smaple); // 对所有进行预测
        // 预测结果与原结果对比，相等为 1， 不等为 0
        r = std::abs(r - labels.at<int>(i)) <= FLT_EPSILON ? 1.f : 0.f;

    if (i < trainNum)
        {
            train_hr += r; // 累计正确数
        }
        else
        {
            test_hr += r;
        }
    }

    test_hr /= samplesNum - trainNum;
    train_hr = trainNum > 0 ? train_hr / trainNum : 1.;
    cout << "train accuracy :  " << train_hr * 100. << "
";
    cout << "test accuracy :  " << test_hr * 100. << "
";


    // 保存 ocr  模型
    string model_path = "ocr.xml";
    model->save(model_path);
    // 载入模型
    // Ptr<KNearest> knn = KNearest::load<KNearest>(model_path);

    
    waitKey(1);
    return 0;
}

　　2.2.2 使用 knn 识别字符

　　　　

// Mat target_img  为目标图像矩阵
model->save(model_path);
// 载入模型
Ptr<KNearest> knn = KNearest::load<KNearest>(model_path);
float it_type = knn->predict(target_img)

　　　　

以上就是车牌识别的核心代码了。 

全部流程的代码我放到下面这个群里面了，欢迎来交流下载。

广州 OpenCV 学校交流群： 892083812

　　

 参考：

深入理解 OpenCV 

https://www.cnblogs.com/denny402/p/5032839.html