zoukankan      html  css  js  c++  java
  • caffe特征提取/C++数据格式转换

    Caffe生成的数据分为2种格式:Lmdb 和 Leveldb
    • 它们都是键/值对(Key/Value Pair)嵌入式数据库管理系统编程库。
    • 虽然lmdb的内存消耗是leveldb的1.1倍,但是lmdb的速度比leveldb快10%至15%,更重要的是lmdb允许多种训练模型同时读取同一组数据集。
    • 因此lmdb取代了leveldb成为Caffe默认的数据集生成格式。

    create_babyface.sh调用的convertData的源代码如下:


    #include<sys/types.h>
    #include<sys/stat.h>
    #include<dirent.h>
    #include <stdio.h>
    #include<string.h>
    
    #include <fstream>  // NOLINT(readability/streams)
    #include <string>
    #include <vector>
    
    #include "boost/scoped_ptr.hpp"
    #include "glog/logging.h"
    #include "google/protobuf/text_format.h"
    #include "stdint.h"
    
    #include "caffe/proto/caffe.pb.h"
    #include "caffe/util/db.hpp"
    
    #include <opencv/cv.h>
    #include <opencv/highgui.h>
    
    using caffe::Datum;
    using boost::scoped_ptr;
    using std::string;
    namespace db = caffe::db;
    using namespace std;
    
    const int kCIFARSize = 32;
    const int kCIFARChannelBytes = 1024;
    const int kCIFARImageNBytes = 3072;
    const int kCIFARBatchSize = 1000;//1000 for a batch!
    const int kCIFARTrainBatches = 5;
    
    void read_image(std::ifstream* file, int* label, char* buffer) {
        char label_char;
        file->read(&label_char, 1);
        *label = label_char;
        file->read(buffer, kCIFARImageNBytes);
        return;
    }
    
    //Read IPLimage to the buffer
    void read_image(
            IplImage* out, char* buffer,
            char* RC, char* GC, char* BC)
    {
        int x,y;
        int idx =0;
        for(y = 0; y<out->height; y++){
            char *ptr= out->imageData + y * out->widthStep;
            for( x = 0;x< out->width;x++){
                idx =y*out->height + x;
                BC[idx]= ptr[3*x];
                GC[idx]= ptr[3*x+1];
                RC[idx]=  ptr[3*x+2]; //这样就可以添加自己的操作,这里我使三通道颜色一样,就彩色图转黑白图了
            }
        }
        memcpy( buffer ,RC, kCIFARChannelBytes*sizeof(char) );
        memcpy( buffer+ kCIFARChannelBytes*sizeof(char) ,      GC,kCIFARChannelBytes*sizeof(char) );
        memcpy( buffer+ kCIFARChannelBytes*sizeof(char)  *2, BC,kCIFARChannelBytes*sizeof(char) );
        return;
    }
    
    //Travel the folder and load the filelist!
    //使用linux dirent遍历目录
     int traveldir(char* path ,int depth, vector<string > &FileList)
    {
        DIR* d;// a
        struct dirent *file; struct stat  sb;
    
        if( !(d=opendir(path ) ) ){
            printf("Read path %s error,wishchin! ", path);
            return -1;
        }
    
        while( (file= readdir(d ) ) != NULL ) {
            if(0== strncmp(file->d_name,  ".", 1 ) ) continue;
            char filename[256];
            strcpy( filename , file->d_name );
    
            string  Sfilename(filename);string  Spath(path);
            Spath.append(Sfilename);
            FileList.push_back(Spath);
        }
    
        if( stat(file->d_name,  &sb)>=0 && S_ISDIR(sb.st_mode) && depth <=4 )
            traveldir(file->d_name,depth+1,FileList);
    
        closedir(d);
        return 1;
    }
    
    // convert the data to the lmdb format !
    void convert_dataset(
            const string& input_folder,
            const string& output_folder,
            const string& db_type) {
    
        scoped_ptr<db::DB> train_db(db::GetDB(db_type));
        train_db->Open(output_folder + "/babyface_train_" + db_type, db::NEW);
        scoped_ptr<db::Transaction> txn(train_db->NewTransaction());
    
        char* path=new char[256];
        int depth=2;
        vector<string > FileList(0);
    
        // Data buffer
        int label;
        IplImage* ImageS;
        char str_buffer[kCIFARImageNBytes];
        char* RC=new char[kCIFARChannelBytes];
        char* GC=new char[kCIFARChannelBytes];
        char* BC=new char[kCIFARChannelBytes];
        Datum datum;
        datum.set_channels(3);
        datum.set_height(kCIFARSize);
        datum.set_width(kCIFARSize);
    
        //"Writing Training data"//载入训练数据
        LOG(INFO) << "Writing Training data";
    
        strcpy(path,( input_folder+(string)("train1") ).c_str() );
        traveldir( path , depth, FileList);
    
        for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
            // Open files
            LOG(INFO) << "Training Batch " << fileid + 1;
            snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
            //CHECK(data_file) << "Unable to open train file #" << fileid + 1;
    
            label=1;//The Batch has 10000 pics!
            for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
                ImageS =cvLoadImage( (FileList[ fileid*kCIFARTrainBatches + itemid] ).c_str() );
                read_image( ImageS, str_buffer, RC,  GC, BC);
    
                datum.set_label(label);//datum.set_label(label);
                datum.set_data(str_buffer, kCIFARImageNBytes);
    
                int length = snprintf(str_buffer,  kCIFARImageNBytes,
                                      "%05d", fileid * kCIFARBatchSize + itemid);
                string out;
                CHECK(datum.SerializeToString( &out)  )  ;
                txn->Put(string(str_buffer, length),  out);//The main sentence ,put data to the txn!
            }
        }
    
        strcpy(path,( input_folder+(string)("train0") ).c_str() );
        traveldir( path , depth, FileList);
        for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
            LOG(INFO) << "Training Batch " << fileid + 1;
            snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
            //CHECK(data_file) << "Unable to open train file #" << fileid + 1;
    
            label=0;//The Batch has 10000 pics!
            for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
                ImageS =cvLoadImage( (FileList[ fileid*kCIFARTrainBatches + itemid] ).c_str() );
                read_image( ImageS, str_buffer, RC,  GC, BC);
    
                datum.set_label(label);//datum.set_label(label);
                datum.set_data(str_buffer, kCIFARImageNBytes);
    
                int length = snprintf(str_buffer,  kCIFARImageNBytes,
                                      "%05d", fileid * kCIFARBatchSize + itemid);
                string out;
                CHECK(datum.SerializeToString( &out)  )  ;
                txn->Put(string(str_buffer, length),  out);//The main sentence ,put data to the txn!
            }
        }
    
        txn->Commit();
        train_db->Close();
    
        //写入测试数据!
        LOG(INFO) << "Writing Testing data";
        scoped_ptr<db::DB> test_db(db::GetDB(db_type));
        test_db->Open(output_folder + "/babyface_test_" + db_type, db::NEW);
        txn.reset(test_db->NewTransaction());
    
        strcpy(path,( input_folder+(string)("test1") ).c_str() );
        traveldir( path , depth, FileList);
        for (int fileid = 0; fileid < 2; ++fileid) {
            LOG(INFO) << "Training Batch " << fileid + 1;
            snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
    
            label=1;//The Batch has 10000 pics!
            for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
                ImageS =cvLoadImage( (FileList[ fileid*2 + itemid] ).c_str() );
                read_image( ImageS, str_buffer, RC,  GC, BC);
    
                datum.set_label(label);//datum.set_label(label);
                datum.set_data(str_buffer, kCIFARImageNBytes);
    
                int length = snprintf(str_buffer,  kCIFARImageNBytes,
                                      "%05d", fileid * kCIFARBatchSize + itemid);
                string out;
                CHECK(datum.SerializeToString( &out)  )  ;
                txn->Put(string(str_buffer, length),  out);//The main sentence ,put data to the txn!
            }
        }
    
        strcpy(path,( input_folder+(string)("test0") ).c_str() );
        traveldir( path , depth, FileList);
        for (int fileid = 0; fileid < 2; ++fileid) {
    
            LOG(INFO) << "Training Batch " << fileid + 1;
            snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
    
            label=0;//The Batch has 10000 pics!
            for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
                ImageS =cvLoadImage( (FileList[ fileid*2 + itemid] ).c_str() );
                read_image( ImageS, str_buffer, RC,  GC, BC);
    
                datum.set_label(label);//datum.set_label(label);
                datum.set_data(str_buffer, kCIFARImageNBytes);
    
                int length = snprintf(str_buffer,  kCIFARImageNBytes,
                                      "%05d", fileid * kCIFARBatchSize + itemid);
                string out;
                CHECK(datum.SerializeToString( &out)  )  ;
                txn->Put(string(str_buffer, length),  out);//The main sentence ,put data to the txn!
            }
        }
    
        txn->Commit();
        test_db->Close();
    
        cvReleaseImage(&ImageS);
        delete [] RC;delete [] GC;delete [] BC;
    }
    
    int main(int argc, char** argv) {
        if (argc != 4) {
            printf("This script converts the CIFAR dataset to the leveldb format used
    "
                   "by caffe to perform classification.
    "
                   "Usage:
    "
                   "    convert_cifar_data input_folder output_folder db_type
    "
                   "Where the input folder should contain the binary batch files.
    "
                   "The CIFAR dataset could be downloaded at
    "
                   "    http://www.cs.toronto.edu/~kriz/cifar.html
    "
                   "You should gunzip them after downloading.
    ");
        } else {
            google::InitGoogleLogging(argv[0]);
            convert_dataset(string(argv[1]), string(argv[2]), string(argv[3]));
        }
        return 0;
    }
    

            后记:目的是载入32×32的三通道图像,直接输入3072维的char向量进行训练,至于怎样训练网络,还得仔细查看一下。

          后记:代码出现 coredump 问题,利用 gcc path/...bin  -o coredemo -g ,出现caffe.pb.h 包含丢失现象,why???

  • 相关阅读:
    SGU 271 Book Pile (双端队列)
    POJ 3110 Jenny's First Exam (贪心)
    HDU 4310 Hero (贪心)
    ZOJ 2132 The Most Frequent Number (贪心)
    POJ 3388 Japanese Puzzle (二分)
    UVaLive 4628 Jack's socks (贪心)
    POJ 2433 Landscaping (贪心)
    CodeForces 946D Timetable (DP)
    Android Studio教程从入门到精通
    Android Tips – 填坑手册
  • 原文地址:https://www.cnblogs.com/wishchin/p/9200183.html
Copyright © 2011-2022 走看看