数据聚合
import pandas as pd
from pandas import Series
import numpy as np
# 准备数据
df = pd.DataFrame([[-0.204708,1.393406,'a','one'],
[0.478943,0.092908,'a','two'],
[-0.519439,0.281746,'b','one'],
[-0.555730,0.769023,'b','two'],
[1.965781,1.246435,'a','one']],columns=['data1','data2','key1','key2'])
df
data1 data2 key1 key2
0 -0.204708 1.393406 a one
1 0.478943 0.092908 a two
2 -0.519439 0.281746 b one
3 -0.555730 0.769023 b two
4 1.965781 1.246435 a one
# 先生成分组对象
grouped = df.groupby('key1')
# 执行具体的运算
grouped.quantile(0.9)
0.9 data1 data2
key1
a 1.668413 1.364012
b -0.523068 0.720295
# 自定义的方法(agg接收自定义函数)这种方式速度要慢的多，不太推荐。
def peak_to_peak(arr):
return arr.max() - arr.min()
grouped.agg(peak_to_peak)
data1 data2
key1
a 2.170489 1.300498
b 0.036291 0.487277
# 聚合运算也可以用describe()
grouped.describe().T
key1 a b
data1 count 3.000000 2.000000
mean 0.746672 -0.537584
std 1.109736 0.025662
min -0.204708 -0.555730
25% 0.137118 -0.546657
50% 0.478943 -0.537584
75% 1.222362 -0.528512
max 1.965781 -0.519439
data2 count 3.000000 2.000000
mean 0.910916 0.525385
std 0.712217 0.344557
min 0.092908 0.281746
25% 0.669671 0.403565
50% 1.246435 0.525385
75% 1.319920 0.647204
max 1.393406 0.769023
经过优化的groupby方法
| 函数名 |
说明 |
| count |
分组中非NA的数量 |
| sum |
非NA的和 |
| mean |
非NA值的平均值 |
| median |
非NA的算术中位数 |
| std、var |
无偏（分母n-1）标准差和方差 |
| min、max |
非NA值的最小值和最大值 |
| prod |
非NA值的积 |
| first、last |
第一个和最后一个非NA值 |
餐馆小费的数据
tips = pd.read_csv('C:/Users/1/Desktop/tips.csv')
tips[:6]
total_bill tip sex smoker day time size
0 16.99 1.01 Female No Sun Dinner 2
1 10.34 1.66 Male No Sun Dinner 3
2 21.01 3.50 Male No Sun Dinner 3
3 23.68 3.31 Male No Sun Dinner 2
4 24.59 3.61 Female No Sun Dinner 4
5 25.29 4.71 Male No Sun Dinner 4
# 小费占总额百分比的列
tips['tip_pct'] = tips['tip']/tips['total_bill']
tips[:6]
total_bill tip sex smoker day time size tip_pct
0 16.99 1.01 Female No Sun Dinner 2 0.059447
1 10.34 1.66 Male No Sun Dinner 3 0.160542
2 21.01 3.50 Male No Sun Dinner 3 0.166587
3 23.68 3.31 Male No Sun Dinner 2 0.139780
4 24.59 3.61 Female No Sun Dinner 4 0.146808
5 25.29 4.71 Male No Sun Dinner 4 0.186240
# 根据sex和smoker对tips进行分组
grouped1 = tips.groupby(['sex','smoker'])
# 直接将讲过优化的groupby方法传入
grouped1['tip_pct'].agg('mean')
sex smoker
Female No 0.156921
Yes 0.182150
Male No 0.160669
Yes 0.152771
Name: tip_pct, dtype: float64
# 如果传入一组函数或函数名，对应的列名就是函数名
grouped1['tip_pct'].agg(['mean','std',peak_to_peak])
mean std peak_to_peak
sex smoker
Female No 0.156921 0.036421 0.195876
Yes 0.182150 0.071595 0.360233
Male No 0.160669 0.041849 0.220186
Yes 0.152771 0.090588 0.674707
# 传入的是一个由(name,function)元组组成的列表，则元组的第一个元素会被当做列名
grouped1['tip_pct'].agg([('foo','mean'),('bar',np.std)])
foo bar
sex smoker
Female No 0.156921 0.036421
Yes 0.182150 0.071595
Male No 0.160669 0.041849
Yes 0.152771 0.090588
grouped['tip_pct','total_bill'].agg(['count','mean','max'])
tip_pct total_bill
count mean max count mean max
sex smoker
Female No 54 0.156921 0.252672 54 18.105185 35.83
Yes 33 0.182150 0.416667 33 17.977879 44.30
Male No 97 0.160669 0.291990 97 19.791237 48.33
Yes 60 0.152771 0.710345 60 22.284500 50.81
# 这里也可以传入带有自定义名称的元组列表,
ftuples = [('new1','mean'),('new2',np.var)]
grouped['tip_pct','total_bill'].agg(ftuples)
tip_pct total_bill
new1 new2 new1 new2
sex smoker
Female No 0.156921 0.001327 18.105185 53.092422
Yes 0.182150 0.005126 17.977879 84.451517
Male No 0.160669 0.001751 19.791237 76.152961
Yes 0.152771 0.008206 22.284500 98.244673
#传入字典
grouped.agg({'tip':np.max,'size':'sum'})
tip size
sex smoker
Female No 5.2 140
Yes 6.5 74
Male No 9.0 263
Yes 10.0 150
# 只有将多个函数应用到至少一列时，DataFrame才会拥有层次化的列
grouped.agg({'tip_pct':['min','max','mean','std'],'size':'sum'})
tip_pct size
min max mean std sum
sex smoker
Female No 0.056797 0.252672 0.156921 0.036421 140
Yes 0.056433 0.416667 0.182150 0.071595 74
Male No 0.071804 0.291990 0.160669 0.041849 263
Yes 0.035638 0.710345 0.152771 0.090588 150
# 以''无索引''的形式返回聚合数据
# 不需要的时候可以关闭所有的索引，包括层次化索引
tips.groupby(['sex','smoker'],as_index=False).mean()
sex smoker total_bill tip size tip_pct
0 Female No 18.105185 2.773519 2.592593 0.156921
1 Female Yes 17.977879 2.931515 2.242424 0.182150
2 Male No 19.791237 3.113402 2.711340 0.160669
3 Male Yes 22.284500 3.051167 2.500000 0.152771