keras+resnet实现车牌识别

1.使用PIL和opencv生成车牌图像数据

from PIL import ImageFont,Image,ImageDraw
import cv2
import numpy as np
import os
from math import *
#创建 生成车牌图像数据 的类
index = {"京": 0, "沪": 1, "津": 2, "渝": 3, "冀": 4, "晋": 5, "蒙": 6, "辽": 7, "吉": 8, "黑": 9, "苏": 10, "浙": 11, "皖": 12,
         "闽": 13, "赣": 14, "鲁": 15, "豫": 16, "鄂": 17, "湘": 18, "粤": 19, "桂": 20, "琼": 21, "川": 22, "贵": 23, "云": 24,
         "藏": 25, "陕": 26, "甘": 27, "青": 28, "宁": 29, "新": 30, "0": 31, "1": 32, "2": 33, "3": 34, "4": 35, "5": 36,
         "6": 37, "7": 38, "8": 39, "9": 40, "A": 41, "B": 42, "C": 43, "D": 44, "E": 45, "F": 46, "G": 47, "H": 48,
         "J": 49, "K": 50, "L": 51, "M": 52, "N": 53, "P": 54, "Q": 55, "R": 56, "S": 57, "T": 58, "U": 59, "V": 60,
         "W": 61, "X": 62, "Y": 63, "Z": 64}
 
chars = ["京", "沪", "津", "渝", "冀", "晋", "蒙", "辽", "吉", "黑", "苏", "浙", "皖", "闽", "赣", "鲁", "豫", "鄂", "湘", "粤", "桂",
             "琼", "川", "贵", "云", "藏", "陕", "甘", "青", "宁", "新", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A",
             "B", "C", "D", "E", "F", "G", "H", "J", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "U", "V", "W", "X",
             "Y", "Z"
             ]
def r(val):
    return int(np.random.random()*val)
def GenCh(f,val):#生成中文字符，f是中文字体对象，
    img=Image.new('RGB',(45,70),(255,255,255))#创建中文字符区域的全白画布，中文一般要方正一些，所以画布设置大一点，等下再resize
    draw=ImageDraw.Draw(img)#对画布创建画画对象
    draw.text((0,3),val,(0,0,0),font=f)#画画对象画出规定字体的黑色的规定val【0】文字,在左上角位置是（0，3）的点开始
    img =  img.resize((23,70))
    A=np.array(img)#图画转换成array格式
    return A
def GenCh1(f,val):#生成英文和数字字符，f是中文字体对象，
    img=Image.new('RGB',(23,70),(255,255,255))
    draw=ImageDraw.Draw(img)
    draw.text((0,2),val,(0,0,0),font=f)#画画对象在左上角（0，2）出开始画出val，颜色全黑，字体是f
    A=np.array(img)
    return A
def rot(img,angle,shape,max_angle):#透视畸变 
    size_o=[shape[1],shape[0]]#cv读的（h,w)换成[w,h]
    size=(shape[1]+int(shape[0]*cos((float(max_angle)/180)*3.14)),shape[0])#【变化后的w，h】
    interval=abs(int(sin(float(angle)/180)*3.14)*shape[0])#h变换的绝对值
    pts1=np.float32([[0,0],[0,size_o[1]],[size_o[0],0],[size_o[0],size_o[1]]])#(00,0h,w0,wh)
    if (angle>0):
        pts2=np.float32([[interval,0],[0,size[1]],[size[0],0],[size[0]-interval,size_o[1]]])
    else:
        pts2 = np.float32([[0,0],[interval,size[1]  ],[size[0]-interval,0  ],[size[0],size_o[1]]])
    M=cv2.getPerspectiveTransform(pts1,pts2)#根据两幅图的四个坐标点计算透视矩阵
    dst=cv2.warpPerspective(img,M,size)#img再M矩阵的变化下生成size大小的变化图
    return dst
def rotRandrom(img,factor,shape):#仿射畸变
    pts1=np.float32([[0,0],[0,shape[0]],[shape[1],0],[shape[1],shape[0]]])#00,0h,w0,wh
    pts2=np.float32([[r(factor),r(factor)],[r(factor),shape[0]-r(factor)],[shape[1]-factor,r(factor)],[shape[1]-r(factor),shape[0]-r(factor)]])
    M=cv2.getPerspectiveTransform(pts1,pts2)
    dst=cv2.warpPerspective(img,M,shape)
    return dst
def tfactor(img):#添加饱和度光照的噪声
    hsv=cv2.cvtColor(img,cv2.COLOR_BGR2HSV)
    hsv[:,:,0] = hsv[:,:,0]*(0.8+ np.random.random()*0.2)
    hsv[:,:,1] = hsv[:,:,1]*(0.3+ np.random.random()*0.7)
    hsv[:,:,2] = hsv[:,:,2]*(0.2+ np.random.random()*0.8)
    img=cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
    return img
def random_envirment(img,data_set):#添加自然环境噪声
    index=r(len(data_set))
    env=cv2.imread(data_set[index])#随机读出一张环境噪声图片
    env=cv2.resize(env,(img.shape[1],img.shape[0]))#改变环境噪声图片的大下
    bak=(img==0)
    bak=bak.astype(np.uint8)*255#图片的黑色部分变成白色部分
    inv=cv2.bitwise_and(bak,env)#图片的非黑部分和噪声求and，
    img=cv2.bitwise_or(inv,img)
    return img
def AddGauss(img,level):#添加高斯模糊
    return cv2.blur(img,(level*2+1,level*2+1))
def AddNoiseSingleChannel(single):#高斯噪声
    diff=255-single.max()
    noise=np.random.normal(0,1+r(6),single.shape)
    noise = (noise - noise.min())/(noise.max()-noise.min())
    noise= diff*noise
    noise= noise.astype(np.uint8)
    dst = single + noise
    return dst
def addNoise(img,sdev = 0.5,avg=10):#每个通道随机添加高斯噪声
    img[:,:,0] =  AddNoiseSingleChannel(img[:,:,0])
    img[:,:,1] =  AddNoiseSingleChannel(img[:,:,1])
    img[:,:,2] =  AddNoiseSingleChannel(img[:,:,2])
    return img

 def AddSmudginess(img, Smu):#模拟车牌的污渍特征
     rows = r(Smu.shape[0] - 50)
     cols = r(Smu.shape[1] - 50)
     adder = Smu[rows:rows + 50, cols:cols + 50];
     adder = cv2.resize(adder, (50, 50));
     #adder = cv2.bitwise_not(adder)
     img = cv2.resize(img,(50,50))
     img = cv2.bitwise_not(img)
     img = cv2.bitwise_and(adder, img)
     img = cv2.bitwise_not(img)
     return img
class GenPlate:#生成车牌图像数据 的类
    def __init__(self,fontCh,fontEng,NoPlates):
        self.fontC=ImageFont.truetype(fontCh,43,0)#创建中文字体对象，fontCh是字体文件地址，43是字体大小,规定文字字体
        self.fontE=ImageFont.truetype(fontEng,60,0)
        self.img=np.array(Image.new('RGB',(226,70),(255,255,255)))#创建（226，70）大小的全白图像，并转换成array格式
        self.bg=cv2.resize(cv2.imread('./input_data/images/template.bmp'),(226,70))#读取出背景模板图
        self.smu=cv2.imread('./input_data/images/smu2.jpg')#********************************
        self.noplates_path=[]
        for parent,parent_folder,filenames in os.walk(NoPlates):#NoPlates的上级目录，NoPlates的子目录（没有），NoPlates的子文件
            for filename in filenames:
                path=parent+'/'+filename
                self.noplates_path.append(path)#环境噪声图片
    def draw(self,val):
        offset=2
        self.img[0:70,offset+8:offset+8+23]=GenCh(self.fontC,val[0])#再self.img画布上画出中文字
        self.img[0:79,offset+8+23+6:offset+8+23+6+23]=GenCh1(self.fontE,val[1])#英文字
        for i in range(5):#画出5个数字
            base=offset+8+23+6+23+17+ i*23+ i*6
            self.img[0:70,base:base+23]=GenCh1(self.fontE,val[i+2])
        return self.img#画出背景白色，文字黑色的文字内容
    def generate(self,text):
        if len(text)==7:
            fg=self.draw(text)#调用draw函数，画出这7个字符)
            fg=cv2.bitwise_not(fg)#图像的array按位取反：黑色背景，白色文字
            com=cv2.bitwise_or(fg,self.bg)#再与背景图片取或，则生成背景是背景图片，前景是白色文字的图片array格式
            com=rot(com,r(60)-30,com.shape,30)#透视畸变效果(img,angle,shape,max_angle)
            com = rotRandrom(com,10,(com.shape[1],com.shape[0]))#仿射畸变
            com = tfactor(com)#添加饱和度光照的噪声
            com = random_envirment(com,self.noplates_path)#环境图片的噪声
            com = AddGauss(com, 1+r(4))#高斯模糊
            com = addNoise(com)#高斯噪声
            return com
    def genPlateString(self,pos,val):#随机生成（中文 英文 数字*5）的字符
        #pos!=-1时，读取出val值就是车牌值************************pos=-1时，val随机，主要是生成车牌
        plateStr=''
        if (pos!=-1):#
            plateStr+=val#读出你想要生成的车牌号
        else:#随机生成车牌号
            for cpos in range(7):
                if cpos==0:
                    plateStr+=chars[r(31)]
                elif cpos==1:
                    plateStr+=chars[41+r(24)]
                else:
                    plateStr+=chars[31+r(10)]
        return plateStr
    def genBatch(self,batchSize,size):#生成batch——size个车牌数据
        for i in range(batchSize):
            plateStr=self.genPlateString(-1,-1)
            print(plateStr)
            img=self.generate(plateStr)
            
            img=cv2.resize(img,size)
            filename=str(plateStr)+'.jpg'
            cv2.imencode('.jpg',img)[1].tofile(filename)#此处解决中文名乱码
if __name__=='__main__':
    G=GenPlate("./input_data/font/platech.ttf",'./input_data/font/platechar.ttf',"./input_data/NoPlates")
    G.genBatch(10,(224,224))

2.使用keras生成resnet34模型

#2.构建网络模型
from keras.models import Model
from keras.layers import Input,Dropout,BatchNormalization,Conv2D,MaxPooling2D,AveragePooling2D,concatenate,Activation,ZeroPadding2D,add,Flatten,Dense
import numpy as np
seed=1
np.random.seed(seed)
import matplotlib.pyplot as plt
%matplotlib inline
characters="京沪津渝冀晋蒙辽吉黑苏浙皖闽赣鲁豫鄂湘粤桂琼川贵云藏陕甘青宁新0123456789ABCDEFGHJKLMNPQRSTUVWXYZ"
width,height,n_len,n_classes=224,224,7,len(characters)
def gen(batch_size=32):
    X = np.zeros((batch_size, height, width, 3), dtype=np.uint8)
    y = [np.zeros((batch_size, n_classes), dtype=np.uint8) for i in range(n_len)]
   # generator = ImageCaptcha(width=width, height=height)
    while True:
        for i in range(batch_size):
            a=G.genPlateString(-1,-1)
            b=G.generate(a)
            img = cv2.resize(b,(224,224))
            X[i] = img.astype('float32')
            for j, ch in enumerate(a):
                y[j][i, :] = 0
                y[j][i, characters.find(ch)] = 1
        yield X, y

def decode(y):
    y = np.argmax(np.array(y), axis=2)[:,0]
    return ''.join([characters[x] for x in y])
 
"""
使用者三段代码测试生成结果，以及解码结果
X, y = next(gen(1))
plt.imshow(X[0])
print(decode(y))
plt.title(decode(y))

"""
def gen(batch_size=32):#生成数据
    x=np.zeros((batch_size,height,width,3),dtype=np.uint8)
    #y=[np.zeros((batch_size,n_classes),dtype=np.uint8) for i in range (n_len)]
    y=np.zeros((batch_size,n_len,n_classes),dtype=np.uint8)
    for i in range(batch_size):
        string=G.genPlateString(-1,1)
        img=G.generate(string)
        img=cv2.resize(img,(224,224))
        x[i]=img.astype('float32')
        for j,ch in enumerate(string):
            y[i][j,:]=0
            y[i][j,characters.find(ch)]=1
    yield x,y
def decode(y):

    y=np.argmax(np.array(y),axis=2)[0]
    return ''.join([characters[x] for x in y])
#x,y=next(gen(1))
#plt.imshow(x[0])
#print(decode(y))
#plt.title(decode(y))
#resnet34网络
def Conv2d_BN(x,nb_filter,kernel_size,strides=(1,1),padding='same',name=None):
    if name is not None:
        bn_name=name+'_bn'
        conv_name=name+'_conv'
    else:
        bn_name=None
        conv_name=None
    x=Conv2D(nb_filter,kernel_size,padding=padding,strides=strides,activation='relu',name=conv_name)(x)
    x=BatchNormalization(name=bn_name)(x)
    return x
def Conv_Block(input_layer,nb_filter,kernel_size,strides=(1,1),with_conv_shortcut=False):
    x=Conv2d_BN(input_layer,nb_filter=nb_filter,kernel_size=kernel_size,strides=strides,padding='same')
    x=Conv2d_BN(x,nb_filter=nb_filter,kernel_size=kernel_size,padding='same')
    if with_conv_shortcut:#残差边
        shortcut=Conv2d_BN(input_layer,nb_filter=nb_filter,strides=strides,kernel_size=kernel_size)
        x=add([x,shortcut])#残差边卷积后相加,缩小网络输出图像尺寸
        return x
    else:
        x=add([x,input_layer])#残差边直接相加，加深网络卷积深度
        return x
input_x=Input(shape=(224,224,3))
x=ZeroPadding2D((3,3))(input_x)
x=Conv2d_BN(x,nb_filter=64,kernel_size=(7,7),strides=(2,2),padding='valid')
x=MaxPooling2D(pool_size=(3,3),strides=(2,2),padding='same')(x)#56，56，64
x = Conv_Block(x,nb_filter=64,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=64,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=64,kernel_size=(3,3))  #56，56，34
x = Conv_Block(x,nb_filter=128,kernel_size=(3,3),strides=(2,2),with_conv_shortcut=True)  
x = Conv_Block(x,nb_filter=128,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=128,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=128,kernel_size=(3,3)) #28，28，128
x = Conv_Block(x,nb_filter=256,kernel_size=(3,3),strides=(2,2),with_conv_shortcut=True)  
x = Conv_Block(x,nb_filter=256,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=256,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=256,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=256,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=256,kernel_size=(3,3))  #14，14，256
x = Conv_Block(x,nb_filter=512,kernel_size=(3,3),strides=(2,2),with_conv_shortcut=True)  
x = Conv_Block(x,nb_filter=512,kernel_size=(3,3))  
x = Conv_Block(x,nb_filter=512,kernel_size=(3,3))#7，7，512  
x = AveragePooling2D(pool_size=(7,7))(x) #1,1,512
x=Flatten()(x)#512
#x = Dense(1000,activation='softmax')(x)  
x = [Dense(n_classes, activation='softmax', name='P%d'%(i+1))(x) for i in range(7)]#有7个输出结果，所以7个Dense层和x分别连接
model = Model(inputs=input_x,outputs=x)  
model.compile(loss='categorical_crossentropy',optimizer='sgd',metrics=['accuracy'])  
model.summary() 
model.fit_generator(gen(), epochs=1000, steps_per_epoch=20,validation_data=gen(),validation_steps=1)        
#保存模型图
from keras.utils import plot_model
plot_model(model, to_file='model.png',show_shapes='True')
#测试下结果
X, y = next(gen(1))
y_pred = model.predict(X)
print(X)
print("ddddd")
print(X[0])
print(decode(y))
print(decode(y_pred))
plt.title('real: %s
pred:%s'%(decode(y), decode(y_pred)))
plt.imshow(X[0], cmap='gray')
#这里显示是和正常的颜色不一样，这是因为，plt读取的通道顺序和cv2的通道顺序是不同的
#保存模型和参数
model.save('resnet34_model.h5')
score=model.evaluate(X,y,verbose=0)
print("test score=",score[0])
print("test accuracy=",score[1])

3.利用模型进行实际预测输出

#进行实际预测
from keras.models import load_model
model=load_model("resnet34_model.h5")
print("导入模型完成")
print("读取图片")
#pic = Image.open("./宁R46974.jpg")
#pic.show()
img = cv2.imread('./x.jpg')#地址不能有中文
img=img[np.newaxis,:,:,:]#图片是三维的但是训练时是转换成4维了所以需要增加一个维度
predict = model.predict(img)
print("车牌号为：",decode(predict))

参考：https://github.com/Haveoneriver/License-Plate-Recognition-Items

https://github.com/szad670401/end-to-end-for-chinese-plate-recognition