Spark MLlib之使用Breeze操作矩阵向量

这下面的练习中，需要自己将spark的jar包 添加进来。

1.spark Mlib 底层使用的向量、矩阵运算使用了Breeze库。

scalaNLP 是一套 机器学习和数值技算的库。它主要是关于科学技术（sc）、机器学习（ML）和自然语言处理（NLP)的。它包括三个库，Breeze、Epic 和 Puck。

Breeze :是机器学习和数值技术库 ，它是sparkMlib的核心，包括线性代数、数值技术和优化，是一种通用、功能强大、有效的机器学习方法。

Epic     :是一种高性能能统计分析器和结构化预测库

Puck    :是一个快速GPU加速解析器

在使用Breeze 库时，需要导入相关包：

import breeze.linalg._

import breeze.numerics._

　　

具体练习如下：

package leaning

import breeze.linalg._
import breeze.numerics._
import breeze.stats.distributions.Rand

/**
  * Created by dy9776 on 2017/12/5.
  */


object Practise_breeze{
  def main(args: Array[String]) {
    val matrix: DenseMatrix[Double] = DenseMatrix.zeros[Double](3,2)
    println(matrix)
    /*
      0.0  0.0
      0.0  0.0
      0.0  0.0
    */
    //全0向量
    val testVector: DenseVector[Double] = DenseVector.zeros[Double](2)
    println(testVector)

    //全1向量
    val allOneVector=DenseVector.ones[Double](2)
    println(allOneVector)


    //按数值填充向量
    val haveNumberFill =DenseVector.fill[Double](3,2)
    println(haveNumberFill)

    //生成随机向量
    val rangeNUm= DenseVector.range(1,  10 , 2)//DenseVector(1, 3, 5, 7, 9)
    val rangeNUmD= DenseVector.rangeD(1,  9 , 2)//DenseVector(1.0, 3.0, 5.0, 7.0)
    val rangeNUmF= DenseVector.rangeF(1,  7 , 2)//DenseVector(1.0, 3.0, 5.0)
    println(rangeNUm)
    println(rangeNUmD)
    println(rangeNUmF)

    //单位矩阵
    val unitMatrix=DenseMatrix.eye[Double](4)
//    println(unitMatrix)
     /*
     1.0  0.0  0.0  0.0
     0.0  1.0  0.0  0.0
     0.0  0.0  1.0  0.0
     0.0  0.0  0.0  1.0
      */

    //对角矩阵
    val doubleVecoter=diag(DenseVector(3.0, 4.0 , 5.0))
//    println(doubleVecoter)
    /*
    3.0  0.0  0.0
    0.0  4.0  0.0
    0.0  0.0  5.0
     */

    //按照行创建矩阵
    val byRowCreateMatrix= DenseMatrix( (4.0, 5.0, 6.0 ) , (7.0 ,8.0 ,9.0))
//    println(byRowCreateMatrix)
    /*
    4.0  5.0  6.0
    7.0  8.0  9.0
     */

    //按照行创建向量
    val denseCreateVector = DenseVector((4.0, 5.0, 6.0, 7.0, 8.0, 9.0))
//    println(denseCreateVector) ///DenseVector((4.0,5.0,6.0,7.0,8.0,9.0)

    //向量装置
    val vectorTranspostion= DenseVector( (4.0, 5.0, 6.0, 7.0, 8.0, 9.0) ).t
    println(vectorTranspostion)//Transpose(DenseVector((4.0,5.0,6.0,7.0,8.0,9.0)))

    //从函数创建向量
    val funCreateVector=DenseVector.tabulate(5)(i=> i*i)
    println(funCreateVector)//DenseVector(0, 1, 4, 9, 16)
    val funCreateVector2=DenseVector.tabulate( 0 to 5)(i=> i*i)
    println(funCreateVector2)//DenseVector(0, 1, 4, 9, 16, 25)

    //从函数创建矩阵
    val createFuncMatrix= DenseMatrix.tabulate(3, 4) {
      case (i ,j ) => i*i + j*j
    }
//    println(createFuncMatrix)
    /*
       0  1  4  9
       1  2  5  10
       4  5  8  13
     */

    //从数组创建矩阵
    val createFunctionMatrix= new DenseMatrix[Double](3, 2, Array(1.0, 4.0, 7.0, 3.0, 6.0, 9.0))
//    println(createFunctionMatrix)
    /*
    1.0  3.0
    4.0  6.0
    7.0  9.0
     */

    //0 到 1的随机向量
    val formZeroToOneRandomVector= DenseVector.rand( 9, Rand.uniform)
    println(formZeroToOneRandomVector)
//    DenseVector(0.7978222133507369, 0.48978247271729325, 0.24943434133065834, 0.6619447026155139, 0.5324942068054981, 0.9051865626036415, 0.5989291014099107, 0.9221881029987078, 0.17371486701192662)
    val formZeroToOneRandomVector2= DenseVector.rand( 9, Rand.uniform)
    println(formZeroToOneRandomVector2)
//DenseVector(0.7978222133507369, 0.48978247271729325, 0.24943434133065834, 0.6619447026155139, 0.5324942068054981, 0.9051865626036415, 0.5989291014099107, 0.9221881029987078, 0.17371486701192662)


    //0 到 1 的随机矩阵
    val formZeroToOneRandomMatrix= DenseMatrix.rand(3, 2, Rand.uniform)
    println(formZeroToOneRandomMatrix)
    /*
    0.8036324612618653  0.538112087890035
    0.6864375371630702  0.3123993272549075
    0.9458628172312897  0.01137554621536796
     */
    val formZeroToOneRandomMatrix2=DenseMatrix.rand(3, 2, Rand.gaussian)
    println(formZeroToOneRandomMatrix2)
    /*
      0.9510499901472648   0.287812938654061
      -0.5266499883462216  0.9380426076781263
      -0.3959295333472151  -0.9057610233257112
     */

    //Breeze元素访问
    val a = new DenseVector[Int](Array(1 to 20 : _*))
    println(a)//DenseVector(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)

    //指定位置

    println(a(0)) //1

    //向量子集
    println( a(1 to 4) )//DenseVector(2, 3, 4, 5)
    println( a(1 until 4) )//DenseVector(2, 3, 4)

    //指定开始位置至结尾
    println( a(1 to -1) )//DenseVector(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)

    //按照指定步长去子集  这个是倒序方式
    println( a(5 to 0 by -1) )//DenseVector(6, 5, 4, 3, 2, 1)


    //最后一个元素
    println( a(-1)) //20


    val m = DenseMatrix((1.0, 2.0, 3.0), (4.0, 5.0, 6.0))
    println(m)
    /*
    1.0  2.0  3.0
    4.0  5.0  6.0
     */

    //指定位置
    println( m(0 ,1) ) //2.0

    //矩阵指定列
    println( m(:: ,1) ) // DenseVector(2.0, 5.0)


//Breeze元素操作

    //调整矩阵形状
    val justAdjustMatrix =m.reshape(3, 2)
    println(justAdjustMatrix)
    /*
    DenseVector(2.0, 5.0)
    1.0  5.0
    4.0  3.0
    2.0  6.0
     */

    //矩阵转成向量
    val toVector=m.toDenseVector
    println(toVector)//DenseVector(1.0, 4.0, 2.0, 5.0, 3.0, 6.0)
    println(toVector.toDenseMatrix)//1.0  4.0  2.0  5.0  3.0  6.0

    //复制下三角
    println(lowerTriangular(m))//
    /*
    1.0  0.0
    4.0  5.0
     */

    //复制上三角
    println(upperTriangular(m))
    /*
    1.0  2.0
    0.0  5.0
     */

    //矩阵复制
    println(m.copy)
    //m 原始为这样的
    //    1.0  2.0
    //    0.0  5.0


    //取对角线元素
    println(diag(upperTriangular(m)))
    //DenseVector(1.0, 5.0)   个人觉得很怪异，不是应该为(1.0, 6.0)吗？上面上、下三角 也好像出乎的意料


    //子集赋数值
    println(a(1 to 4 ):=5)
    //(1.0, 5.0)

    //子集赋向量
    println( a(1 to 4):=DenseVector(1,2,3,4) )
      //DenseVector(1, 2, 3, 4)

    println(m)
    //矩阵赋值
//    println( m( 1 to 2, 1 to 2) := 0.0 )
//Exception in thread "main" java.lang.IndexOutOfBoundsException: Row slice of Range(1, 2) was bigger than matrix rows of 2
    println("-==========m1================-")
    println( m( 0 to 1, 1 to 2) := 0.0 )
    println("-==========m================-")
    println(m)
    println("-==========m end================-")
    /*
    -==========m1================-
    0.0  0.0
    0.0  0.0
    -==========m================-
    0.0  0.0  3.0
    0.0  0.0  6.0
    -==========m end================-
    */

    //矩阵列赋值
    val re=m(::, 2) := 5.0
    println(re.toDenseMatrix)
    //5.0  5.0


    val a1 = DenseMatrix((1.0, 2.0, 3.0), (4.0, 5.0, 6.0))
    val a2 = DenseMatrix((7.0, 8.0, 9.0), (10.0, 11.0, 12.0))


    //垂直连接矩阵
    val verticalLike=DenseMatrix.vertcat(a1, a2)
    println(verticalLike)
    println("-==========================-")
    /*
    1.0   2.0   3.0
    4.0   5.0   6.0
    7.0   8.0   9.0
    10.0  11.0  12.0
     */

    //横向连接矩阵
    val twoMatrixConn=DenseMatrix.horzcat( a1, a2)
    println(twoMatrixConn)
    println("-==========================-")
/*
1.0  2.0  3.0  7.0   8.0   9.0
4.0  5.0  6.0  10.0  11.0  12.0
 */

    //向量的连接
    val connnectVector1=DenseVector.vertcat(DenseVector(20, 21, 22), DenseVector(23, 24, 25))
    val connnectVector2=DenseVector.horzcat(DenseVector(20, 21, 22), DenseVector(23, 24, 25))

    println(connnectVector1)//DenseVector(20, 21, 22, 23, 24, 25)
    println(connnectVector2)
    /*
      20  23
      21  24
      22  25
     */


//Breeze数值计算函数
    //元素加法
    println(a1 + a2)
    /*
      8.0   10.0  12.0
      14.0  16.0  18.0
     */

    //元素乘法
    println(a1 :* a2)
    /*
      7.0   16.0  27.0
      40.0  55.0  72.0
     */

    //元素除法
    println(a1 :/ a2)
    /*
      0.14285714285714285  0.25                 0.3333333333333333
      0.4                  0.45454545454545453  0.5
     */

    //元素比较
    println(a1 :< a2)
    /*
      true  true  true
      true  true  true
     */

    //元素相等
    println(a1 :== a2)
    /*
      false  false  false
      false  false  false
     */

    //元素追加
    println(a1 :+=2.0)
    /*
      3.0  4.0  5.0
      6.0  7.0  8.0
     */

    //元素追乘
    println(a1 :*=2.0)
    /*
      6.0   8.0   10.0
      12.0  14.0  16.0
     */

    //向量点积
    val vectorDot=DenseVector(1, 2, 3, 4) dot DenseVector(1, 1, 1, 1)
    println(vectorDot)//10

    //元素最大值
    println(max(a1))//16.0

    //元素最小值
    println(min(a1))//6.0

    //元素最大值的位置
    println(argmax(a1))// (1,2)

    //元素最小值的位置
    println(argmin(a1))// (0,0)

    //Breeze求和函数

    val m1 = DenseMatrix((1.0, 2.0, 3.0, 4.0), (5.0, 6.0, 7.0, 8.0), (9.0, 10.0, 11.0, 12.0))
    println(m1)
    /*
        1.0  2.0   3.0   4.0
        5.0  6.0   7.0   8.0
        9.0  10.0  11.0  12.0
     */

    println("-==========================-")
    //元素求和
    println(sum(m1))//78.0

    //每一列求和
    println(sum(m1, Axis._0))//res59: breeze.linalg.DenseMatrix[Double] = 15.0  18.0  21.0  24.0

    //每一行求和
    println(sum(m1, Axis._1))//res60: breeze.linalg.DenseVector[Double] = DenseVector(10.0, 26.0, 42.0)

    //对角线元素和
     println(trace(lowerTriangular(m1)))// res61: Double = 18.0

    //累积和
    val a3 = new DenseVector[Int](Array(10 to 20: _*))
    println(accumulate(a3)) // DenseVector(10, 21, 33, 46, 60, 75, 91, 108, 126, 145, 165)


//Breeze布尔函数

    val c = DenseVector(true, false, true)
    val d = DenseVector(false, true, true)
    //元素与操作
    println(c :& d) // DenseVector(false, false, true)

    //元素或操作
    println(c :| d) //DenseVector(true, true, true)

    //元素非操作
    println(!c) //DenseVector(false, true, false)


    val e = DenseVector[Int](-3, 0, 2)


    //存在非零元素
    println(any(e)) //true

    //所有元素非零
    println(all(e)) //false

//Breeze线性代数函数
    val f = DenseMatrix((1.0, 2.0, 3.0), (4.0, 5.0, 6.0), (7.0, 8.0, 9.0))
    val g = DenseMatrix((1.0, 1.0, 1.0), (1.0, 1.0, 1.0), (1.0, 1.0, 1.0))

    //线性求解，AX = B，求解X
    println(f  g)
      /* breeze.linalg.DenseMatrix[Double] =
        -2.5  -2.5  -2.5
        4.0   4.0   4.0
        -1.5  -1.5  -1.5
       */

    //转置
    println(f.t)
    /* breeze.linalg.DenseMatrix[Double] =
      1.0  4.0  7.0
      2.0  5.0  8.0
      3.0  6.0  9.0
     */

    //求特征值
    println(det(f)) // Double = 6.661338147750939E-16

    //求逆
    println(inv(f))
    /*
      -4.503599627370499E15  9.007199254740992E15    -4.503599627370495E15
      9.007199254740998E15   -1.8014398509481984E16  9.007199254740991E15
      -4.503599627370498E15  9.007199254740992E15    -4.5035996273704955E15
     */

    //求伪逆
    println(pinv(f))
    /*
      -3.7720834019330525E14  7.544166803866101E14    -3.77208340193305E14
      7.544166803866094E14    -1.5088333607732208E15  7.544166803866108E14
      -3.772083401933041E14   7.544166803866104E14    -3.772083401933055E14
     */

    //特征值和特征向量
    println(eig(f))
    /*
    Eig(DenseVector(16.116843969807043, -1.1168439698070427, -1.3036777264747022E-15),DenseVector(0.0, 0.0, 0.0),-0.23197068724628617  -0.7858302387420671   0.40824829046386363
        -0.5253220933012336   -0.08675133925662833  -0.816496580927726
         -0.8186734993561815   0.61232756022881      0.4082482904638625
        )
     */


    //奇异值分解
    val svd.SVD(u,s,v) = svd(g)
    println(u)
    /*
      -0.5773502691896255  -0.5773502691896257  -0.5773502691896256
      -0.5773502691896256  -0.2113248654051871  0.7886751345948126
      -0.5773502691896256  0.7886751345948129   -0.21132486540518708
     */
    println("==============================")
    println(s) //DenseVector(3.0000000000000004, 0.0, 0.0)
    println("==============================")
    println(v)
    /*
      -0.5773502691896256  -0.5773502691896257  -0.5773502691896256
      0.0                  -0.7071067811865474  0.7071067811865477
      0.816496580927726    -0.4082482904638629  -0.4082482904638628
     */


    //求矩阵的秩
    println(rank(f))  //2

    //矩阵长度
    println(f.size) //9

    //矩阵行数
    println(f.rows) // 3

    //矩阵列数
    f.cols // 3


//Breeze取整函数

    val h = DenseVector(-1.2, 0.7, 2.3) // breeze.linalg.DenseVector[Double] = DenseVector(-1.2, 0.7, 2.3)

    //四舍五入
    println( round(h)  ) // breeze.linalg.DenseVector[Long] = DenseVector(-1, 1, 2)

    //大于它的最小整数
    println( ceil(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(-1.0, 1.0, 3.0)

    //小于它的最大整数
    println( floor(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(-2.0, 0.0, 2.0)

    //符号函数
    println( signum(h) ) // breeze.linalg.DenseVector[Double] = DenseVector(-1.0, 1.0, 1.0)

    //取正数
    println( abs(h) )  // breeze.linalg.DenseVector[Double] = DenseVector(1.2, 0.7, 2.3)


  }

}