Pandas DataFrame四种写入方法效率对比

该测试程序是将protocol变量转换为DataFrame类型为了便于使用numpy等数学工具
ImuEncode_pb2 是使用protocol生成的model， 具体使用方法可参考Protocol buffer 指南。

这是我使用的一些变量的声明

import time 
import ImuEncode_pb2
import numpy as np
import pandas as pd 
from pandas.core.frame import DataFrame
from ImuSignal import LegString
from ImuSignal import LegPositionString
from ImuSignal import SignalAttributeDict
from OnlineClassifier import OnlineClassifier

ImuSignalDataset = ImuEncode_pb2.ImuSignalDataset()
for i in range(50):
	imusignal =ImuSignalDataset.imusignal.add()
	for j in range(4):
		signal = imusignal.signal.add()
		signal.TimeStamp = 0
		signal.acc.AccX = 0
		signal.acc.AccY = 0
		signal.acc.AccZ = 0
		signal.angle.Roll = 0
		signal.angle.Pitch = 0
		signal.anglerate.Course = 0
		signal.quaternion.v = 0
		signal.quaternion.x = 0
		signal.quaternion.y = 0
		signal.quaternion.z = 0
		signal.leg = ImuEncode_pb2.Signal.RT
		signal.legposition = ImuEncode_pb2.Signal.Foot

rate = 100 #sample rate
period = 0.5 #classify period
Leg = ['RT','LT']
Leg_Position = ['Foot','Thigh']
angle = ['Roll','Pitch','Course']
accel = ['AccX','AccY','AccZ']

特别注意下面代码段的加入，这是对于内存的声明，我是随便赋值不知道Python变量的声明原理后续完善

SignalFrame = DataFrame()
for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

该方法不简单并且耗时应该是最长的，测试耗时为： 0.09793972969055176

SignalFrame = DataFrame()
for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

starttime = time.time()
SignalFrame = pd.concat([pd.DataFrame(data={"Foot_AccX_RT":imusignal.signal[0].acc.AccX,  "Foot_AccY_RT":imusignal.signal[0].acc.AccY,        "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,
            "Foot_Roll_RT":imusignal.signal[0].angle.Roll,  "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch,    "Foot_Course_RT":imusignal.signal[0].anglerate.Course,
            "Thigh_AccX_RT":imusignal.signal[1].acc.AccX,   "Thigh_AccY_RT":imusignal.signal[1].acc.AccY,       "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,
            "Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch,   "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,
            "Foot_AccX_LT":imusignal.signal[2].acc.AccX,    "Foot_AccY_LT":imusignal.signal[2].acc.AccY,        "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,
            "Foot_Roll_LT":imusignal.signal[2].angle.Roll,  "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch,    "Foot_Course_LT":imusignal.signal[2].anglerate.Course,
            "Thigh_AccX_LT":imusignal.signal[3].acc.AccX,   "Thigh_AccY_LT":imusignal.signal[3].acc.AccY,       "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,
            "Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch,   "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,
        },index=[0]) for imusignal in ImuSignalDataset.imusignal],ignore_index=True)
endtime = time.time()
print("cost time:",endtime-starttime)

该方法不简单并且耗时长，测试耗时为： 0.04797029495239258

SignalFrame = DataFrame()

for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

starttime = time.time()
for imusignal in ImuSignalDataset.imusignal:
    SignalFrame = SignalFrame.append({"Foot_AccX_RT":imusignal.signal[0].acc.AccX,  "Foot_AccY_RT":imusignal.signal[0].acc.AccY,        "Foot_AccZ_RT":imusignal.signal[0].acc.AccZ,
            "Foot_Roll_RT":imusignal.signal[0].angle.Roll,  "Foot_Pitch_RT":imusignal.signal[0].angle.Pitch,    "Foot_Course_RT":imusignal.signal[0].anglerate.Course,
            "Thigh_AccX_RT":imusignal.signal[1].acc.AccX,   "Thigh_AccY_RT":imusignal.signal[1].acc.AccY,       "Thigh_AccZ_RT":imusignal.signal[1].acc.AccZ,
            "Thigh_Roll_RT":imusignal.signal[1].angle.Roll, "Thigh_Pitch_RT":imusignal.signal[1].angle.Pitch,   "Thigh_Course_RT":imusignal.signal[1].anglerate.Course,
            "Foot_AccX_LT":imusignal.signal[2].acc.AccX,    "Foot_AccY_LT":imusignal.signal[2].acc.AccY,        "Foot_AccZ_LT":imusignal.signal[2].acc.AccZ,
            "Foot_Roll_LT":imusignal.signal[2].angle.Roll,  "Foot_Pitch_LT":imusignal.signal[2].angle.Pitch,    "Foot_Course_LT":imusignal.signal[2].anglerate.Course,
            "Thigh_AccX_LT":imusignal.signal[3].acc.AccX,   "Thigh_AccY_LT":imusignal.signal[3].acc.AccY,       "Thigh_AccZ_LT":imusignal.signal[3].acc.AccZ,
            "Thigh_Roll_LT":imusignal.signal[3].angle.Roll, "Thigh_Pitch_LT":imusignal.signal[3].angle.Pitch,   "Thigh_Course_LT":imusignal.signal[3].anglerate.Course,
        },ignore_index=True)
endtime = time.time()
print("cost time:",endtime-starttime)

该方法简单并且耗时仅长于最快的实现方法，测试耗时为：0.016989469528198242

SignalFrame = DataFrame()

for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000
starttime = time.time()
i=0
for imusignal in ImuSignalDataset.imusignal:
    for SignalData in imusignal.signal:
        signal_dict = SignalAttributeDict(SignalData)
        for signal in angle + accel:
            SignalFrame.at[i,LegPositionString(SignalData.legposition)+"_"+signal+"_"+LegString(SignalData.leg)]=signal_dict[signal]
    i+=1
endtime = time.time()
print("cost time:",endtime-starttime)

该方法不简单，如果数据量较大可能会影响数据的准确度，但是耗时应该是最短的，测试耗时为：0.012990951538085938

SignalFrame = DataFrame()

for leg in Leg:
    for leg_position in  Leg_Position:
        for signal in  accel + angle:
            SignalFrame[leg_position+"_"+signal+"_"+leg]  = np.arange(999999999,999999999+int(period*rate),1)/1000000

starttime = time.time()
i=0
for imusignal in ImuSignalDataset.imusignal:
    SignalFrame.loc[i]=[imusignal.signal[0].acc.AccX   , imusignal.signal[0].acc.AccY    , imusignal.signal[0].acc.AccZ,
                            imusignal.signal[0].angle.Roll  , imusignal.signal[0].angle.Pitch , imusignal.signal[0].anglerate.Course,
                            imusignal.signal[1].acc.AccX    , imusignal.signal[1].acc.AccY    , imusignal.signal[1].acc.AccZ,
                            imusignal.signal[1].angle.Roll  , imusignal.signal[1].angle.Pitch , imusignal.signal[1].anglerate.Course,
                            imusignal.signal[2].acc.AccX    , imusignal.signal[2].acc.AccY    , imusignal.signal[2].acc.AccZ,
                            imusignal.signal[2].angle.Roll  , imusignal.signal[2].angle.Pitch , imusignal.signal[2].anglerate.Course,
                            imusignal.signal[3].acc.AccX    , imusignal.signal[3].acc.AccY    , imusignal.signal[3].acc.AccZ,
                            imusignal.signal[3].angle.Roll  , imusignal.signal[3].angle.Pitch , imusignal.signal[3].anglerate.Course]
    i+=1
endtime = time.time()
print("cost time:",endtime-starttime)

用于生成ImuEncode_pb2的ImuEncode.proto文件如下

syntax = "proto2";
package ImuTutorial;

message Signal {
  enum Leg {
    RT = 0;
    LT = 1;
  }
  enum LegPosition{
    Foot = 0;
    Thigh = 1;
  }

  required Leg leg = 1 [default = RT];
  required LegPosition legposition = 2 [default = Foot];
  optional float TimeStamp = 3;
  optional int64 time = 4;
  optional int32 id = 5;

  message Acc{
    required float AccX = 1;
    required float AccY = 2;
    required float AccZ = 3;
  }
  message Angle{
    optional float Course = 1;
    required float Roll = 2;
    required float Pitch = 3;
  }
  message AngleRate{
    required float Course = 1;
    optional float Roll = 2;
    optional float Pitch = 3;
  }
  message Quaternion{
    optional float v = 1;
    required float x = 2;
    required float y = 3;
    required float z = 4;
  }
  required Acc acc = 6;
  required Angle angle= 7;
  required AngleRate anglerate= 8;
  optional Quaternion quaternion = 9;
}

message ImuSignal {
  required uint64 time = 1;
  repeated Signal signal = 2;  
}

message ImuSignalDataset {
  repeated ImuSignal imusignal = 1;
}

message Datalength {
  required fixed32 length = 1;
}