2.5&2.6 numpy&pandas 笔记

https://www.bilibili.com/video/av16378934?from=search&seid=9658415144190341605

numpy.random模块 https://blog.csdn.net/fireflychh/article/details/73603838

pandas && numpy

1. 属性以及创建矩阵

import numpy as np
array = np.array([[1,2,3],
                  [2,3,4]]) #建立一个矩阵(注意'[]'的数量)
print(array)
print('number of dim', array.ndim)  #矩阵的维数
print('shape', array.shape)    #矩阵的形状
print('size', array.size)   #矩阵的大小，数据个数

a = np.array([2,23,4], dtype=np.int) ##int32，32位int    @1
a1 = np.zeros((3,4), dtype=np.int32)  #3行4列的全为0, 32位int类型数据
a2 = np.ones((3,4), dtype=np.int32)  #3行4列的全为1, 32位int类型数据
a3 = np.arange(10, 20, 2)  #生成从10到20(取不到20)的步长为2的数列
a4 = np.arange(12).reshape((3,4))  #生成从0到11的，3行4列的矩阵
a5 = np.linspace(1, 10, 5)    #生成线段，生成从1到10一共5段线段
#  Out[13]: array([ 1.  ,  3.25,  5.5 ,  7.75, 10.  ])

2. 基础运算

import numpy as np
a = np.array([10,20,30,40])
b = np.arange(4)   #[0,1,2,3]
c = a - b #减法
d = a + b #加法
e = a * b #乘法注意不是矩阵乘法
f = b**2  #次方
g = 10*np.sin(a) #sin与cos用法一样
print(b < 3)    #[ True  True  True False]

a1 = np.array([[1,1],
               [0,1]])
b1 = np.arange(4).reshape((2,2))
c1 = a1 * b1 #逐个相乘    array([[0, 1],
                               # [0, 3]])
c1_dot = np.dot(a1, b1) ##矩阵乘法  array([[2, 4],
                                       #  [2, 3]])
c1_dot_2 = a.dot(b) ##另一种写法 个人觉得可读性较差
a3 =
a2 = np.random.random((2,4))#array([[0.41322246, 0.70201529, 0.3669404 , 0.24802252],
                               #[0.3022681 , 0.89514217, 0.63173225, 0.57511299]])
x = np.sum(a2, axis=1)  #求和，axis=1表示以取行元素操作列， axis=0表示以列取元素操作行,Out[24]: array([1.73020067, 2.4042555 ])
x1 = np.min(a2, axis=0) #最小值 array([0.3022681 , 0.70201529, 0.3669404 , 0.24802252])
x2 = np.max(a2, axis=1) #最大值array([0.70201529, 0.89514217])

import numpy as np
a = np.arange(2, 14).reshape((3,4))
print(np.argmin(a)) ##最小值的索引位置  0
print(np.argmax(a))  ##最大值的索引位置  11
print('平均值',np.mean(a)) ##平均值 或者  a.mean()
print(np.average(a))  ##平均值
print('中位数',np.median(a)) ##中位数 7.5
print('累加',np.cumsum(a))  ##累加依次相邻两个数之和 [ 2  5  9 14 20 27 35 44 54 65 77 90]
print('累差',np.diff(a))  ##相邻两个数之间的差值
#累差 [[1 1 1]
#[1 1 1]
#[1 1 1]]
print('输出非零数据的行数和列数',np.nonzero(a))
#行数：(array([0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2], dtype=int64),
#列数： array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3], dtype=int64))
print('转置',np.transpose(a)) ##矩阵转置
print('类似二值化',np.clip(a, 5, 9)) ##类似二值化 小于5的数变成5，大于9的数变成9
#[[5 5 5 5]
# [6 7 8 9]
# [9 9 9 9]]   如果数据相反就只有大于5的变5而不是小于9的变9

a1 = np.array([[1,4,2,7],
              [3,1,5,8],
               [8,3,5,6]])
print("排序", np.sort(a1)) ##注意逐行排序
# [[1 2 4 7]
# [1 3 5 8]
# [3 5 6 8]]

3.numpy的索引

import numpy as np
a = np.arange(3,15).reshape((3,4))
##如果是数列的话直接a[n]
print(a[2]) ##输出第二行所以数据
print('等同于',a[2, :])
print(a[2][2]) ##输出第二行第二列
print(a[2, 1:3])##第二行第一列到第二列
##迭代行
for raw in a:
    print(raw)
##迭代列
for column in a.T: ##a.T 相当于 np.transpose(a) a的转置
    print(column)
##迭代项
print('将矩阵转换成一行的序列', a.flatten()) ## 返回序列[ 3  4  5  6  7  8  9 10 11 12 13 14]
for item in a.flat: ##flat() 返回的是一个生成序列的迭代器
    print(item)

4.numpy 合并

import numpy as np
a = np.array([1,1,1])
b = np.array([2,2,2])
print('上下的合并',np.vstack((a,b,c))) ## 注意需要括号将合并的数据括起来
#[[1 1 1]
# [2 2 2]]
print('左右合并',np.hstack((a,b,c)))  ##[1 1 1 2 2 2]

##注意transpose无法将序列转换为列
#需要如下操作
a1 = np.array([1,1,1])[:,np.newaxis] #或者reshape(n,1)也可以
a2 = np.array([2,2,2])[:,np.newaxis]
print(np.hstack((a1,a2)))
#[[1 2]
# [1 2]
# [1 2]]
print('总的合并方法(在newaxis方法下）',np.concatenate((a1,a2),axis=1))  ##(在newaxis方法下）才能使用，1行0列

就newaxis方法进行如下解释

首先newaxis作用是增加维度

详解：np.newaxis在[]中第几位，a.shape的第几维就变成1，a的原来的维度依次往后排。
例子：若a.shape=(a ,b, c)
a[:, np.newaxis].shape= (a, 1, b, c)
a[:, np.newaxis, np.newaxis].shape= (a, 1, 1, b, c)
a[np.newaxis, :].shape= (1, a, b, c)
a[np.newaxis, np.newaxis, :].shape= (1, 1, a, b, c)
a[np.newaxis, :, np.newaxis].shape= (1, a, 1, b, c)
a[np.newaxis, :, np.newaxis, :].shape= (1, a, 1, b, c)

a[:, :,np.newaxis].shape= (a, b,1, c)

另外

np.newaxis=None

a[np.newaxis, :, np.newaxis, :].shape == a[None,:,None,:]

如下是具体例子：

import numpy as np
a = np.array([1,1,1])
a1 = a[:,np.newaxis]
b = np.array([2,2,2])
b1 = b[np.newaxis,:]
print(a.shape, a)
print(a1.shape)
print(a1)
print(b.shape, b)
print(b1.shape)
print(b1)

结果

(3,) [1 1 1] 
(3, 1)
[[1]
[1]
[1]]
(3,) [2 2 2]
(1, 3)
[[2 2 2]]

c = np.array([[1,1,1],[2,2,2]])
c1 = c[:,np.newaxis]
d = np.array([[2,2,2],[3,3,3]])
d1 = d[np.newaxis,:]
print(c.shape)
print(c)
print(c1.shape)
print(c1)
print(d.shape)
print(d)
print(d1.shape)
print(d1）
结果

(2, 3)
[[1 1 1]
 [2 2 2]]
(2, 1, 3)
[[[1 1 1]]

 [[2 2 2]]]
(2, 3)
[[2 2 2]
 [3 3 3]]
(1, 2, 3)
[[[2 2 2]
  [3 3 3]]]

5.numpy的array分割

import numpy as np
a = np.arange(12).reshape((3,4))
print(a)
##没有整除会报ValueError
print(np.split(a, 2, axis=1))  ##将a按照列数平均分成两份，每取一行分成两半
print(np.split(a, 3, axis=0))  ##将a按照行数平均分成3分，每取一列分成3分
#[array([[0, 1, 2, 3]]), array([[4, 5, 6, 7]]), array([[ 8,  9, 10, 11]])]

##不等量分割
print(np.array_split(a,2,axis=0))
#[array([[0, 1, 2, 3],
#       [4, 5, 6, 7]]), array([[ 8,  9, 10, 11]])]
print(np.array_split(a,[1,3],axis=1))##按照第一份分0列，第二份分1,2列，第三份分第3列([]相当于切片，索引)
print('上下分割也就是按照行分割',np.vsplit(a,3))
print('左右分割也就是按照列数分割',np.hsplit(a,2))

6. nump的copy以及deep_copy

In [1]: import numpy as np
In [2]: a = np.arange(4)
In [3]: b = a
In [4]: a
Out[4]: array([0, 1, 2, 3])
In [5]: b
Out[5]: array([0, 1, 2, 3])
In [6]: a[0] = 11
In [7]: a
Out[7]: array([11,  1,  2,  3])
In [8]: b
Out[8]: array([11,  1,  2,  3])
In [9]: b is a
Out[9]: True
In [10]: b[1:3] = [22,33]
In [11]: b
Out[11]: array([11, 22, 33,  3])
In [12]: a
Out[12]: array([11, 22, 33,  3])
In [13]: c = a.copy() ##deep_copy
In [14]: a[3] = 44
In [15]: a
Out[15]: array([11, 22, 33, 44])
In [16]: c
Out[16]: array([11, 22, 33,  3])

pandas

1.  基本介绍

 

import numpy as np
import pandas as pd
s = pd.Series([1,3,6,np.nan,44,1]) ##列表，会自动加序号
print('产生列表',s)
dates = pd.date_range('20200206',periods=6)
##产生日期，periods表示生成的个数
#DatetimeIndex(['2020-02-06', '2020-02-07', '2020-02-08', '2020-02-09',
#               '2020-02-10', '2020-02-11'],
#              dtype='datetime64[ns]', freq='D')
df = pd.DataFrame(np.random.randn(6,4), index=dates, columns=['a','b','r','t'])
## 其中randn之后会讲到， index表示行的名称，column表示列的名称
#                   a         b         r         t
#2020-02-06 -0.661183  0.586394  0.867207  0.234598
#2020-02-07  2.050175  0.962423  0.412010  1.479072
#2020-02-08  1.132787  1.256786 -0.234434 -0.860920
#2020-02-09 -0.118119  1.201684 -1.485074  0.460520
#2020-02-10 -0.040067 -0.871771 -0.720332  0.055199
#2020-02-11 -0.165296 -1.925641  0.002060  0.701225
df1 = pd.DataFrame(np.arange(12).reshape((3,4)))
##默认名称就是0-n-1的数字
#   0  1   2   3
#0  0  1   2   3
#1  4  5   6   7
#2  8  9  10  11
data = pd.Series(np.random.randn(1000),index=np.arange(1000))
data = data.cumsum()#元素累加和，也是1000个数据



#字典的表示方法
df2 = pd.DataFrame({'A':[1.,2.,3.,4.],
                    'B':pd.date_range('20200206',periods=4),
                    'C':pd.Series(np.arange(4)),
                    'D':pd.Categorical(['test','train','train','tets']),
                    'E':'foo'})
#     A          B  C      D    E
#0  1.0 2020-02-06  0   test  foo
#1  2.0 2020-02-07  1  train  foo
#2  3.0 2020-02-08  2  train  foo
#3  4.0 2020-02-09  3   tets  foo
print('打印数据类型',df2.dtypes)
print('打印列序号的名字',df2.columns)#打印列序号的名字 Index(['A', 'B', 'C', 'D', 'E'], dtype='object')
print('打印每个数据',df2.values)
print(df2.describe())
print('转置',df2.T)
print('排序',df2.sort_index(axis=1, ascending=False))
##倒序排序，按照行取值 ascend上升
print('按照某列数据排序', df2.sort_values(by='C', ascending=False))
##按照'C'列数据进行排序

 

 

 

2.  数据选择

import pandas as pdi
import numpy as np

dates = pd.date_range('20200206', periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)), index=dates, columns=['A','B','C','D'])
方法1
df.A 与 df['A']一样
df[0:3], df['20200206':'20200208]
方法2
select by label:loc
df.loc['20200207']
df.loc['20200207',['A','B']]
方法3
select by position: iloc
df.iloc[1,1]
df.iloc[[1,3,5],1:3]
方法4
Boolean indexing：
df[df.A>8]

3. 设置值

import pandas as pd
import numpy as np

dates = pd.date_range('20200206', periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)), index=dates, columns=['A','B','C','D'])
#改变值
df.iloc[2,2] = 111
df.loc['20200207','B'] = 222
df[df.A > 4] = 0
df.B[df.A > 4] = 0
df['E'] = np.nan ##nan无穷大
df['F'] = pd.Series(np.arange(6), index=dates)

4. 处理丢失数据

import pandas as pd
import numpy as np

dates = pd.date_range('20200206', periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)), index=dates, columns=['A','B','C','D'])
df.iloc[0,1] = np.nan
df.iloc[1,2] = np.nan

df.dropna(axis=0, how='any')
# axis=0按照行取数据删掉列
# how={'any', 'all'}  any表示出现nan就删掉，all表示全部一行或者一列是nan删掉
df.fillna(value=0)#将nan数据填入数据0
df.isnull() #判断是否有丢失的数据
#                A      B      C      D
#2020-02-06  False   True  False  False
#2020-02-07  False  False   True  False
#2020-02-08  False  False  False  False
#2020-02-09  False  False  False  False
#2020-02-10  False  False  False  False
#2020-02-11  False  False  False  Fals
np.any(df.isnull()) == True ##检查是否有值丢失

5.导入导出数据

读取

 

保存

 

  

data = pd.read_csv('C:/Users/XXXX/Pictures/untitled3/student.csv')
print(data)
data.to_pickle('C:/Users/XXXX/Desktop/studstudent.pickle')

6.合并

1.concat方法

import pandas as pd
import numpy as np

df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2, columns=['a','b','c','d'])

##concatenation
res = pd.concat([df1, df2, df3], axis=1, ignore_index=True)
#0上下合并，1左右合并
#ignore_index=True 表示忽略行列的名称

df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'], index=[1,2,3])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d','e'], index=[2,3,4])

res1 = pd.concat([df1,df2], join='inner',ignore_index=True)
res2 = pd.concat([df1, df2], join='outer',ignore_index=True)
##默认是outer，如果名字大小不一样全用NaN填充
#inner则不同的数据全部裁剪掉
print(res1)
print(res2)
#结果
     b    c    d
0  0.0  0.0  0.0
1  0.0  0.0  0.0
2  0.0  0.0  0.0
3  1.0  1.0  1.0
4  1.0  1.0  1.0
5  1.0  1.0  1.0
     a    b    c    d    e
0  0.0  0.0  0.0  0.0  NaN
1  0.0  0.0  0.0  0.0  NaN
2  0.0  0.0  0.0  0.0  NaN
3  NaN  1.0  1.0  1.0  1.0
4  NaN  1.0  1.0  1.0  1.0
5  NaN  1.0  1.0  1.0  1.0

res = pd.concat([df1,df2], axis=1, join_axes=[df1.index])
#join_axes表示只考虑df1.index中行名称（删掉了4）
     a    b    c    d    b    c    d    e
1  0.0  0.0  0.0  0.0  NaN  NaN  NaN  NaN
2  0.0  0.0  0.0  0.0  1.0  1.0  1.0  1.0
3  0.0  0.0  0.0  0.0  1.0  1.0  1.0  1.0

2. append方法

res = df1.append([df2,df3], ignore_index=True)

 3. merge方法

 

import pandas as pd
import numpy as np

left = pd.DataFrame({'key':['K0','K1','K2','K3'],
                     'A':['A0','A1','A2','A3'],
                     'B':['B0','B1','B2','B3']})
right = pd.DataFrame({'key':['K0','K1','K2','K3'],
                     'A':['C0','C1','C2','C3'],
                     'D':['D0','D1','D2','D3']})

res = pd.merge(left, right, on='key') #基于'key'的合并
  A_x   B key A_y   D
0  A0  B0  K0  C0  D0
1  A1  B1  K1  C1  D1
2  A2  B2  K2  C2  D2
3  A3  B3  K3  C3  D3
res = pd.merge(left, right,on='key', suffixes=['_boy', '_girl'], how='outer')
##suffixes在相同的index后加上后缀
  A_boy   B key A_girl   D
0    A0  B0  K0     C0  D0
1    A1  B1  K1     C1  D1
2    A2  B2  K2     C2  D2
3    A3  B3  K3     C3  D3


left = pd.DataFrame({'key1':['K0','K0','K1','K2'],
                     'key2':['K0','K1','K0','K1'],
                     'A':['A0','A1','A2','A3'],
                     'B':['B0','B1','B2','B3']})
right = pd.DataFrame({'key1':['K0','K1','K1','K2'],
                      'key2':['K0','K0','K0','K0'],
                     'C':['C0','C1','C2','C3'],
                     'D':['D0','D1','D2','D3']})
res = pd.merge(left, right, on=['key1', 'key2'],how='outer',indicator=True)
#how=['innner', 'outer', 'left', 'right'] 默认inner
#innner ,outer类似上述
#left，right基于左边还是右边的key来进行合并
#indicator显示是否两者都有，默认Flase
     A    B key1 key2    C    D      _merge
0   A0   B0   K0   K0   C0   D0        both
1   A1   B1   K0   K1  NaN  NaN   left_only
2   A2   B2   K1   K0   C1   D1        both
3   A2   B2   K1   K0   C2   D2        both
4   A3   B3   K2   K1  NaN  NaN   left_only
5  NaN  NaN   K2   K0   C3   D3  right_only

res = pd.merge(left, right, left_index=True, right_index=True, how='outer')
#left_index，right_index=True表示用左边右边的index进行合并
    A   B key1_x key2_x   C   D key1_y key2_y
0  A0  B0     K0     K0  C0  D0     K0     K0
1  A1  B1     K0     K1  C1  D1     K1     K0
2  A2  B2     K1     K0  C2  D2     K1     K0
3  A3  B3     K2     K1  C3  D3     K2     K0

 

7.画图（部分）

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

data = pd.Series(np.random.randn(1000),index=np.arange(1000))
data = data.cumsum()
data.plot()
plt.show()

data = pd.DataFrame(np.random.randn(1000,4), columns=list('ABCD'))
data = data.cumsum()
data.plot()
plt.show()

a = data.plot.scatter(x='A', y='B', color='Green')
data.plot.scatter(x='A',y='C', color='Yellow',ax = a)#ax=a将两副图画在一起
plt.show()