Faster RCNN算法训练代码解析（1）

这周看完faster-rcnn后，应该对其源码进行一个解析，以便后面的使用。

那首先直接先主函数出发py-faster-rcnn/tools/train_faster_rcnn_alt_opt.py

我们在后端的运行命令为

python  ./py-faster-rcnn/tools/train_faster_rcnn_alt_opt.py

--gpu
0
--net_name
ZF
--weights
data/imagenet_models/ZF.v2.caffemodel
--imdb
voc_2007_trainval
--cfg
experiments/cfgs/faster_rcnn_alt_opt.yml

从这条命令就可以看出，我们是使用0id的GPU，使用ZF网络，预训练模型使用ZF.v2.caffemodel，数据集使用voc_2007_trainval，配置文件cfg使用faster_rcnn_alt_opt.yml。

先进入主函数：

if __name__ == '__main__':
    args = parse_args() #获取命令行参数
    #Namespace(cfg_file='experiments/cfgs/faster_rcnn_alt_opt.yml', gpu_id=0, imdb_name='voc_2007_trainval', 
    #net_name='ZF', pretrained_model='data/imagenet_models/ZF.v2.caffemodel', set_cfgs=None)
    

　　 print('Called with args:')
    print(args)

    if args.cfg_file is not None: ##配置文件存在，则加载配置文件
        cfg_from_file(args.cfg_file)  ##进入config.py文件，通过yaml加载后使用edict转化格式，然后通过_merge_a_into_b(a, b)迭代融合成一个config
    if args.set_cfgs is not None:
        cfg_from_list(args.set_cfgs)
    cfg.GPU_ID = args.gpu_id  ##设置使用的GPU的id，一般直接为0

    # --------------------------------------------------------------------------
    # Pycaffe doesn't reliably free GPU memory when instantiated nets are
    # discarded (e.g. "del net" in Python code). To work around this issue, each
    # training stage is executed in a separate process using
    # multiprocessing.Process.
    # --------------------------------------------------------------------------

    # queue for communicated results between processes
    mp_queue = mp.Queue()  ##创建一个多线程的对象
    # solves, iters, etc. for each training stage
    solvers, max_iters, rpn_test_prototxt = get_solvers(args.net_name) ##获得solvers等信息

进入get_solvers()函数：

def get_solvers(net_name): ##ZF net
    # Faster R-CNN Alternating Optimization
    n = 'faster_rcnn_alt_opt'  ##采取alt_opt训练方式
    # Solver for each training stage
    solvers = [[net_name, n, 'stage1_rpn_solver60k80k.pt'],
               [net_name, n, 'stage1_fast_rcnn_solver30k40k.pt'],
               [net_name, n, 'stage2_rpn_solver60k80k.pt'],
               [net_name, n, 'stage2_fast_rcnn_solver30k40k.pt']]
    solvers = [os.path.join(cfg.MODELS_DIR, *s) for s in solvers]  ##记录该训练方式的各阶段的solver（训练参数），即rpn训练和整体faster_rcnn训练的slover
    # Iterations for each training stage
    max_iters = [80000, 40000, 80000, 40000] 
    # max_iters = [100, 100, 100, 100]
    # Test prototxt for the RPN
    rpn_test_prototxt = os.path.join(
        cfg.MODELS_DIR, net_name, n, 'rpn_test.pt')  ##记录rpn测试的prototext，即rpn测试时的网络结构
    return solvers, max_iters, rpn_test_prototxt

接着回到主函数里面，开始第一阶段的训练：

　　print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
    print 'Stage 1 RPN, init from ImageNet model'
    print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'

    cfg.TRAIN.SNAPSHOT_INFIX = 'stage1'
    mp_kwargs = dict(
            queue=mp_queue, 
            imdb_name=args.imdb_name,  ##'voc_2007_trainval'
            init_model=args.pretrained_model, ##使用预训练模型'data/imagenet_models/ZF.v2.caffemodel'
            solver=solvers[0],  ##'py-faster-rcnn/models/pascal_voc/ZF/faster_rcnn_alt_opt/stage1_rpn_solver60k80k.pt'
            max_iters=max_iters[0],  ##最大迭代次数80000
            cfg=cfg) 
    p = mp.Process(target=train_rpn, kwargs=mp_kwargs) ##设置进程对象，进程执行train_rpn函数，使用mp_kwargs参数
    p.start()
    rpn_stage1_out = mp_queue.get() ##获取线程中的数据，这里属于进程间的通信
    p.join() ##等待子线性结束

接着进入train_rpn（）函数来看看：

def train_rpn(queue=None, imdb_name=None, init_model=None, solver=None,
              max_iters=None, cfg=None):
    """Train a Region Proposal Network in a separate training process.
    """
    ##注意，第一阶段的训练没有使用任何的建议框，而是使用gt_boxes来训练
    
    cfg.TRAIN.HAS_RPN = True
    cfg.TRAIN.BBOX_REG = False  # 只针对 Fast R-CNN bbox regression来开启该选项
    cfg.TRAIN.PROPOSAL_METHOD = 'gt' #默认使用gt来进行区域建议
    cfg.TRAIN.IMS_PER_BATCH = 1
    print 'Init model: {}'.format(init_model)
    print('Using config:')
    pprint.pprint(cfg)  ##pprint专门打印python数据结构类

    import caffe
    _init_caffe(cfg) ##初始化caffe，设置了随机数种子，以及使用caffe训练时的模式（gpu/cpu）

    roidb, imdb = get_roidb(imdb_name)
    print 'roidb len: {}'.format(len(roidb))
    output_dir = get_output_dir(imdb)
    print 'Output will be saved to `{:s}`'.format(output_dir)

    model_paths = train_net(solver, roidb, output_dir,
                            pretrained_model=init_model,
                            max_iters=max_iters)
    # Cleanup all but the final model
    for i in model_paths[:-1]:
        os.remove(i)
    rpn_model_path = model_paths[-1]
    # Send final model path through the multiprocessing queue
    queue.put({'model_path': rpn_model_path})

 pprint.pprint(cfg)打印出来的config的配置项：

Using config:
{'DATA_DIR': '/home/home/FRCN_ROOT/py-faster-rcnn/data',
 'DEDUP_BOXES': 0.0625,
 'EPS': 1e-14,
 'EXP_DIR': 'faster_rcnn_alt_opt',
 'GPU_ID': 0,
 'MATLAB': 'matlab',
 'MODELS_DIR': '/home/home/FRCN_ROOT/py-faster-rcnn/models/pascal_voc',
 'PIXEL_MEANS': array([[[ 102.9801,  115.9465,  122.7717]]]),
 'RNG_SEED': 3,
 'ROOT_DIR': '/home/home/FRCN_ROOT/py-faster-rcnn',
 'TEST': {'BBOX_REG': True,
          'HAS_RPN': True,
          'MAX_SIZE': 1000,
          'NMS': 0.3,
          'PROPOSAL_METHOD': 'selective_search',
          'RPN_MIN_SIZE': 16,
          'RPN_NMS_THRESH': 0.7,
          'RPN_POST_NMS_TOP_N': 300,
          'RPN_PRE_NMS_TOP_N': 6000,
          'SCALES': [600],
          'SVM': False},
 'TRAIN': {'ASPECT_GROUPING': True,
           'BATCH_SIZE': 128,
           'BBOX_INSIDE_WEIGHTS': [1.0, 1.0, 1.0, 1.0],
           'BBOX_NORMALIZE_MEANS': [0.0, 0.0, 0.0, 0.0],
           'BBOX_NORMALIZE_STDS': [0.1, 0.1, 0.2, 0.2],
           'BBOX_NORMALIZE_TARGETS': True,
           'BBOX_NORMALIZE_TARGETS_PRECOMPUTED': False,
           'BBOX_REG': False,
           'BBOX_THRESH': 0.5,
           'BG_THRESH_HI': 0.5,
           'BG_THRESH_LO': 0.0,
           'FG_FRACTION': 0.25,
           'FG_THRESH': 0.5,
           'HAS_RPN': True,
           'IMS_PER_BATCH': 1,
           'MAX_SIZE': 1000,
           'PROPOSAL_METHOD': 'gt',
           'RPN_BATCHSIZE': 256,
           'RPN_BBOX_INSIDE_WEIGHTS': [1.0, 1.0, 1.0, 1.0],
           'RPN_CLOBBER_POSITIVES': False,
           'RPN_FG_FRACTION': 0.5,
           'RPN_MIN_SIZE': 16,
           'RPN_NEGATIVE_OVERLAP': 0.3,
           'RPN_NMS_THRESH': 0.7,
           'RPN_POSITIVE_OVERLAP': 0.7,
           'RPN_POSITIVE_WEIGHT': -1.0,
           'RPN_POST_NMS_TOP_N': 2000,
           'RPN_PRE_NMS_TOP_N': 12000,
           'SCALES': [600],
           'SNAPSHOT_INFIX': 'stage1',
           'SNAPSHOT_ITERS': 10000,
           'USE_FLIPPED': True,
           'USE_PREFETCH': False},
 'USE_GPU_NMS': True}

继续，现在我们进入函数 roidb, imdb = get_roidb(imdb_name)：

def get_roidb(imdb_name, rpn_file=None):
    imdb = get_imdb(imdb_name)  
    print 'Loaded dataset `{:s}` for training'.format(imdb.name)  ##加载数据完毕
    imdb.set_proposal_method(cfg.TRAIN.PROPOSAL_METHOD)    ##设置区域建议所使用的方法gt，具体使用eval融合字符串再赋值
    print 'Set proposal method: {:s}'.format(cfg.TRAIN.PROPOSAL_METHOD)
    if rpn_file is not None:
        imdb.config['rpn_file'] = rpn_file
    roidb = get_training_roidb(imdb)
    return roidb, imdb

进入imdb = get_imdb(imdb_name)函数，该文件在/py-faster-rcnn/lib/datasets/factory.py，其实主要是运用工厂模式来适配不同的数据集：

for year in ['2007', '2012']:
    for split in ['train', 'val', 'trainval', 'test']:
        name = 'voc_{}_{}'.format(year, split)
        __sets[name] = (lambda split=split, year=year: pascal_voc(split, year))

def get_imdb(name):
    """Get an imdb (image database) by name."""
    if not __sets.has_key(name):
        raise KeyError('Unknown dataset: {}'.format(name))
    return __sets[name]()  ##执行该函数，该函数对应上面的lambda，适配pascal_voc来建造数据

这里其实也是调用了pascal_voc()函数来创建imdb数据，pascal_voc类见py-faster-rcnn/lib/datasets/pascal_voc.py文件中，如下：

class pascal_voc(imdb):
    def __init__(self, image_set, year, devkit_path=None):
        imdb.__init__(self, 'voc_' + year + '_' + image_set)  ##进入基类imdb来进行初始化
        self._year = year
        self._image_set = image_set
        self._devkit_path = self._get_default_path() if devkit_path is None 
                            else devkit_path
        self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year)
        self._classes = ('__background__', # always index 0     该数据集加上背景一共有21类
                         'aeroplane', 'bicycle', 'bird', 'boat',
                         'bottle', 'bus', 'car', 'cat', 'chair',
                         'cow', 'diningtable', 'dog', 'horse',
                         'motorbike', 'person', 'pottedplant',
                         'sheep', 'sofa', 'train', 'tvmonitor')
        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))  ##将各个类随机转化成对应的数字，比如sheep=17
        self._image_ext = '.jpg'
        self._image_index = self._load_image_set_index()  ##读取py-faster-rcnn/data/VOCdevkit2007/VOC2007/ImageSets/Main/trainval.txt
                                                          ##为每个图片标注index，不如000005.jpg=0000

        # Default to roidb handler
        self._roidb_handler = self.selective_search_roidb
        self._salt = str(uuid.uuid4())
        self._comp_id = 'comp4'

        # PASCAL specific config options
        self.config = {'cleanup'     : True,
                       'use_salt'    : True,
                       'use_diff'    : False,
                       'matlab_eval' : False,
                       'rpn_file'    : None,
                       'min_size'    : 2}

        assert os.path.exists(self._devkit_path), 
                'VOCdevkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), 
                'Path does not exist: {}'.format(self._data_path)

这里只截取了一部分，可以发现，pascal_voc这个类主要用来组织输入的图片数据，存储图片的相关信息，但并不存储图片；而实际上，pascal_voc类是imdb类的一个子类；进入imdb的类：

class imdb(object):
    """Image database."""

    def __init__(self, name):
        self._name = name
        self._num_classes = 0
        self._classes = []
        self._image_index = []
        self._obj_proposer = 'selective_search' ##先前的fast rcnn默认使用ss方法进行区域建议
        self._roidb = None
        self._roidb_handler = self.default_roidb
        # Use this dict for storing dataset specific config options
        self.config = {}

    @property   
    def name(self):  ##基类属性在子类（pascal类）创建时若有赋值操作则自动生成
        return self._name

    @property
    def num_classes(self):
        return len(self._classes)

    @property
    def classes(self):
        return self._classes

    @property
    def image_index(self):
        return self._image_index

    @property  ##把方法装饰成该类的属性
    def roidb_handler(self):
        return self._roidb_handler

    @roidb_handler.setter  ##对roidb_handler产生另外一个装饰器，使用setter属性进行赋值
    def roidb_handler(self, val):
        self._roidb_handler = val

    def set_proposal_method(self, method):  ##运用setter来设置训练方法
        method = eval('self.' + method + '_roidb')
        self.roidb_handler = method

    @property
    def roidb(self):
        # A roidb is a list of dictionaries, each with the following keys:
        #   boxes
        #   gt_overlaps
        #   gt_classes
        #   flipped
        if self._roidb is not None:
            return self._roidb
        self._roidb = self.roidb_handler()
        return self._roidb

    @property
    def cache_path(self):
        cache_path = osp.abspath(osp.join(cfg.DATA_DIR, 'cache'))
        if not os.path.exists(cache_path):
            os.makedirs(cache_path)
        return cache_path

    @property
    def num_images(self):
      return len(self.image_index)

 此时我们看看现在的变量值：

好了现在imdb数据已经获得了，再回到get_roidb()里面的imdb = get_imdb(imdb_name)函数中，紧接着set_proposal_method()函数设置了产生proposal的方法，实际也是向imdb中添加roidb数据：

    def set_proposal_method(self, method):
        method = eval('self.' + method + '_roidb')
        self.roidb_handler = method  ##method=self.gt_roidb,这里其实是调用了pascal_voc.py文件里面的gt_roidb()函数

首先用eval()对这个方法进行解析，使其有效，再传入roidb_handler中，这里就要回到之前的train_rpn()函数中了，它里面设置了cfg.TRAIN.PROPOSAL_METHOD='gt'（默认值是selective search,先前用于fast rcnn的），先进入gt_roidb()函数中：

    def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')  ##如果存在gt框的位置文件则加载并返回gt框的信息（roidb）
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = [self._load_pascal_annotation(index)  ##如果不存在则直接读取文件的
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote gt roidb to {}'.format(cache_file)

        return gt_roidb

这里的gt_roidb = [self._load_pascal_annotation(index)函数为：

    def _load_pascal_annotation(self, index):
        """
        Load image and bounding boxes info from XML file in the PASCAL VOC
        format.
        """
        filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
        tree = ET.parse(filename)  ##从硬盘导入xml文件
        objs = tree.findall('object')  ##找到object的tag
        if not self.config['use_diff']:  ##取出tag为difficult的object
            # Exclude the samples labeled as difficult
            non_diff_objs = [
                obj for obj in objs if int(obj.find('difficult').text) == 0]
            # if len(non_diff_objs) != len(objs):
            #     print 'Removed {} difficult objects'.format(
            #         len(objs) - len(non_diff_objs))
            objs = non_diff_objs
        num_objs = len(objs)

        boxes = np.zeros((num_objs, 4), dtype=np.uint16)  ##boxes的存储坐标，4个，所以为四列
        gt_classes = np.zeros((num_objs), dtype=np.int32)  ##gt框的类
        overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)  ##重叠率矩阵
        # "Seg" area for pascal is just the box area
        seg_areas = np.zeros((num_objs), dtype=np.float32)  ##面积

        # Load object bounding boxes into a data frame.
        for ix, obj in enumerate(objs):
            bbox = obj.find('bndbox')
            # Make pixel indexes 0-based
            x1 = float(bbox.find('xmin').text) - 1
            y1 = float(bbox.find('ymin').text) - 1
            x2 = float(bbox.find('xmax').text) - 1
            y2 = float(bbox.find('ymax').text) - 1
            cls = self._class_to_ind[obj.find('name').text.lower().strip()]
            boxes[ix, :] = [x1, y1, x2, y2]
            gt_classes[ix] = cls
            overlaps[ix, cls] = 1.0
            seg_areas[ix] = (x2 - x1 + 1) * (y2 - y1 + 1)

        overlaps = scipy.sparse.csr_matrix(overlaps)

        return {'boxes' : boxes,
                'gt_classes': gt_classes,
                'gt_overlaps' : overlaps,
                'flipped' : False,
                'seg_areas' : seg_areas}

由上面可以看出roidb的结构是一个包含有5个key的字典。

这个时候就从imdb获得了最初的roidb格式的数据，但这还不是训练时的roidb数据，再回到get_roidb()函数中，通过get_training_roidb(imdb)函数得到最终用于训练的roidb数据，进入该函数：

def get_training_roidb(imdb):
    """Returns a roidb (Region of Interest database) for use in training."""
    if cfg.TRAIN.USE_FLIPPED: 
        print 'Appending horizontally-flipped training examples...'
        imdb.append_flipped_images()  ##如果设置了翻转项，则对图片进行水平翻转后添加，原来5000张图片，加入翻转后为10000左右，这里可以理解成数据增强
        print 'done' 

    print 'Preparing training data...'
    rdl_roidb.prepare_roidb(imdb) ##对roidb加入额外的信息，方便训练
    print 'done'

    return imdb.roidb

进入翻转函数append_flipped_images()：

    def append_flipped_images(self):
        num_images = self.num_images
        widths = self._get_widths() ##具体里面是使用PIL库来获取width
        for i in xrange(num_images):
            boxes = self.roidb[i]['boxes'].copy()
            oldx1 = boxes[:, 0].copy()
            oldx2 = boxes[:, 2].copy()
            boxes[:, 0] = widths[i] - oldx2 - 1
            boxes[:, 2] = widths[i] - oldx1 - 1
            assert (boxes[:, 2] >= boxes[:, 0]).all()
            entry = {'boxes' : boxes,
                     'gt_overlaps' : self.roidb[i]['gt_overlaps'],
                     'gt_classes' : self.roidb[i]['gt_classes'],
                     'flipped' : True}
            self.roidb.append(entry)
        self._image_index = self._image_index * 2

进入rdl_roidb.prepare_roidb(imdb)函数：

def prepare_roidb(imdb):
    """Enrich the imdb's roidb by adding some derived quantities that
    are useful for training. This function precomputes the maximum
    overlap, taken over ground-truth boxes, between each ROI and
    each ground-truth box. The class with maximum overlap is also
    recorded.
    """
    sizes = [PIL.Image.open(imdb.image_path_at(i)).size
             for i in xrange(imdb.num_images)]
    roidb = imdb.roidb
    for i in xrange(len(imdb.image_index)):  ##加入位置，宽，高等信息
        roidb[i]['image'] = imdb.image_path_at(i)
        roidb[i]['width'] = sizes[i][0]
        roidb[i]['height'] = sizes[i][1]
        # need gt_overlaps as a dense array for argmax
        gt_overlaps = roidb[i]['gt_overlaps'].toarray()
        # max overlap with gt over classes (columns)
        max_overlaps = gt_overlaps.max(axis=1)   
        # gt class that had the max overlap
        max_classes = gt_overlaps.argmax(axis=1)  
        roidb[i]['max_classes'] = max_classes  ##加入最大概率类
        roidb[i]['max_overlaps'] = max_overlaps  ##加入最大重叠率
        # sanity checks
        # max overlap of 0 => class should be zero (background)
        zero_inds = np.where(max_overlaps == 0)[0]
        assert all(max_classes[zero_inds] == 0)
        # max overlap > 0 => class should not be zero (must be a fg class)
        nonzero_inds = np.where(max_overlaps > 0)[0]
        assert all(max_classes[nonzero_inds] != 0)

查看此时roidb的结构：

此时roidb的图片000005.jpg的，也即index为00000的图片的数据结构下有：boxes、flipped(是否翻转过)、gt_classes、gt_overlaps、height、image、max_classes、max_overlaps、seg_areas（boxes的面积）、width、__len__

到这里为止，我们已经成功利用工厂模式适配pascal_voc的数据集，并读取xml文件来获取数据集的gt框（roisdb），第一部分介绍完毕。