Spark2 oneHot编码--标准化--主成分--聚类

1.导入包

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.Dataset
import org.apache.spark.sql.Row
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.Column
import org.apache.spark.sql.DataFrameReader
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.Encoder
import org.apache.spark.sql.functions._
import org.apache.spark.sql.DataFrameStatFunctions
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.feature.StringIndexer
import org.apache.spark.ml.feature.OneHotEncoder
import org.apache.spark.ml.feature.VectorAssembler
import org.apache.spark.ml.feature.MinMaxScaler
import org.apache.spark.ml.feature.StandardScaler
import org.apache.spark.ml.feature.PCA
import org.apache.spark.ml.clustering.KMeans

2.导入数据

   
val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate() 
   
// For implicit conversions like converting RDDs to DataFrames 
import spark.implicits._
   
val data: DataFrame = spark.read.format("csv").option("header", true).load("hdfs://ns1/datafile/wangxiao/Affairs.csv") 
data: org.apache.spark.sql.DataFrame = [affairs: string, gender: string ... 7 more fields] 
   
data.cache 
res0: data.type = [affairs: string, gender: string ... 7 more fields] 
  
data.limit(10).show() 
+-------+------+---+------------+--------+-------------+---------+----------+------+ 
|affairs|gender|age|yearsmarried|children|religiousness|education|occupation|rating| 
+-------+------+---+------------+--------+-------------+---------+----------+------+ 
|      0|  male| 37|          10|      no|            3|       18|         7|     4| 
|      0|female| 27|           4|      no|            4|       14|         6|     4| 
|      0|female| 32|          15|     yes|            1|       12|         1|     4| 
|      0|  male| 57|          15|     yes|            5|       18|         6|     5| 
|      0|  male| 22|        0.75|      no|            2|       17|         6|     3| 
|      0|female| 32|         1.5|      no|            2|       17|         5|     5| 
|      0|female| 22|        0.75|      no|            2|       12|         1|     3| 
|      0|  male| 57|          15|     yes|            2|       14|         4|     4| 
|      0|female| 32|          15|     yes|            4|       16|         1|     2| 
|      0|  male| 22|         1.5|      no|            4|       14|         4|     5| 
+-------+------+---+------------+--------+-------------+---------+----------+------+ 
   
// 转换字符类型，将Double和String的字段分开放 
val data1 = data.select( 
     |   data("affairs").cast("Double"), 
     |   data("age").cast("Double"), 
     |   data("yearsmarried").cast("Double"), 
     |   data("religiousness").cast("Double"), 
     |   data("education").cast("Double"), 
     |   data("occupation").cast("Double"), 
     |   data("rating").cast("Double"), 
     |   data("gender").cast("String"), 
     |   data("children").cast("String")) 
data1: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 7 more fields] 
   
data1.printSchema() 
root 
 |-- affairs: double (nullable = true) 
 |-- age: double (nullable = true) 
 |-- yearsmarried: double (nullable = true) 
 |-- religiousness: double (nullable = true) 
 |-- education: double (nullable = true) 
 |-- occupation: double (nullable = true) 
 |-- rating: double (nullable = true) 
 |-- gender: string (nullable = true) 
 |-- children: string (nullable = true) 
   
   
data1.limit(10).show 
+-------+----+------------+-------------+---------+----------+------+------+--------+ 
|affairs| age|yearsmarried|religiousness|education|occupation|rating|gender|children| 
+-------+----+------------+-------------+---------+----------+------+------+--------+ 
|    0.0|37.0|        10.0|          3.0|     18.0|       7.0|   4.0|  male|      no| 
|    0.0|27.0|         4.0|          4.0|     14.0|       6.0|   4.0|female|      no| 
|    0.0|32.0|        15.0|          1.0|     12.0|       1.0|   4.0|female|     yes| 
|    0.0|57.0|        15.0|          5.0|     18.0|       6.0|   5.0|  male|     yes| 
|    0.0|22.0|        0.75|          2.0|     17.0|       6.0|   3.0|  male|      no| 
|    0.0|32.0|         1.5|          2.0|     17.0|       5.0|   5.0|female|      no| 
|    0.0|22.0|        0.75|          2.0|     12.0|       1.0|   3.0|female|      no| 
|    0.0|57.0|        15.0|          2.0|     14.0|       4.0|   4.0|  male|     yes| 
|    0.0|32.0|        15.0|          4.0|     16.0|       1.0|   2.0|female|     yes| 
|    0.0|22.0|         1.5|          4.0|     14.0|       4.0|   5.0|  male|      no| 
+-------+----+------------+-------------+---------+----------+------+------+--------+ 
   
val dataDF = data1
dataDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 7 more fields] 
   
dataDF.cache() 
res4: dataDF.type = [affairs: double, age: double ... 7 more fields] 

3.字符转换成数字索引，OneHot编码，注意setDropLast设置为false

字符转换成数字索引 
val indexer = new StringIndexer().setInputCol("gender").setOutputCol("genderIndex").fit(dataDF) 
indexer: org.apache.spark.ml.feature.StringIndexerModel = strIdx_27dba613193a 
   
val indexed = indexer.transform(dataDF) 
indexed: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 8 more fields] 
   
// OneHot编码，注意setDropLast设置为false 
val encoder = new OneHotEncoder().setInputCol("genderIndex").setOutputCol("genderVec").setDropLast(false) 
encoder: org.apache.spark.ml.feature.OneHotEncoder = oneHot_155a53de3aef 
   
val encoded = encoder.transform(indexed) 
encoded: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 9 more fields] 
   
encoded.show() 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+ 
|affairs| age|yearsmarried|religiousness|education|occupation|rating|gender|children|genderIndex|    genderVec| 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+ 
|    0.0|37.0|        10.0|          3.0|     18.0|       7.0|   4.0|  male|      no|        1.0|(2,[1],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     14.0|       6.0|   4.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|32.0|        15.0|          1.0|     12.0|       1.0|   4.0|female|     yes|        0.0|(2,[0],[1.0])| 
|    0.0|57.0|        15.0|          5.0|     18.0|       6.0|   5.0|  male|     yes|        1.0|(2,[1],[1.0])| 
|    0.0|22.0|        0.75|          2.0|     17.0|       6.0|   3.0|  male|      no|        1.0|(2,[1],[1.0])| 
|    0.0|32.0|         1.5|          2.0|     17.0|       5.0|   5.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|22.0|        0.75|          2.0|     12.0|       1.0|   3.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|57.0|        15.0|          2.0|     14.0|       4.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])| 
|    0.0|32.0|        15.0|          4.0|     16.0|       1.0|   2.0|female|     yes|        0.0|(2,[0],[1.0])| 
|    0.0|22.0|         1.5|          4.0|     14.0|       4.0|   5.0|  male|      no|        1.0|(2,[1],[1.0])| 
|    0.0|37.0|        15.0|          2.0|     20.0|       7.0|   2.0|  male|     yes|        1.0|(2,[1],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     18.0|       6.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])| 
|    0.0|47.0|        15.0|          5.0|     17.0|       6.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])| 
|    0.0|22.0|         1.5|          2.0|     17.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     14.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|37.0|        15.0|          1.0|     17.0|       5.0|   5.0|female|     yes|        0.0|(2,[0],[1.0])| 
|    0.0|37.0|        15.0|          2.0|     18.0|       4.0|   3.0|female|     yes|        0.0|(2,[0],[1.0])| 
|    0.0|22.0|        0.75|          3.0|     16.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|22.0|         1.5|          2.0|     16.0|       5.0|   5.0|female|      no|        0.0|(2,[0],[1.0])| 
|    0.0|27.0|        10.0|          2.0|     14.0|       1.0|   5.0|female|     yes|        0.0|(2,[0],[1.0])| 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+ 
only showing top 20 rows 
  
val indexer1 = new StringIndexer().setInputCol("children").setOutputCol("childrenIndex").fit(encoded) 
indexer1: org.apache.spark.ml.feature.StringIndexerModel = strIdx_55db099c07b7
   
val indexed1 = indexer1.transform(encoded) 
indexed1: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 10 more fields] 
   
val encoder1 = new OneHotEncoder().setInputCol("childrenIndex").setOutputCol("childrenVec").setDropLast(false) 
   
val encoded1 = encoder1.transform(indexed1) 
encoded1: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 11 more fields] 
   
encoded1.show() 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+-------------+-------------+ 
|affairs| age|yearsmarried|religiousness|education|occupation|rating|gender|children|genderIndex|    genderVec|childrenIndex|  childrenVec| 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+-------------+-------------+ 
|    0.0|37.0|        10.0|          3.0|     18.0|       7.0|   4.0|  male|      no|        1.0|(2,[1],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     14.0|       6.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|32.0|        15.0|          1.0|     12.0|       1.0|   4.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|57.0|        15.0|          5.0|     18.0|       6.0|   5.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|        0.75|          2.0|     17.0|       6.0|   3.0|  male|      no|        1.0|(2,[1],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|32.0|         1.5|          2.0|     17.0|       5.0|   5.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|22.0|        0.75|          2.0|     12.0|       1.0|   3.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|57.0|        15.0|          2.0|     14.0|       4.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|32.0|        15.0|          4.0|     16.0|       1.0|   2.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|         1.5|          4.0|     14.0|       4.0|   5.0|  male|      no|        1.0|(2,[1],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|37.0|        15.0|          2.0|     20.0|       7.0|   2.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     18.0|       6.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|47.0|        15.0|          5.0|     17.0|       6.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|         1.5|          2.0|     17.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     14.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|37.0|        15.0|          1.0|     17.0|       5.0|   5.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|37.0|        15.0|          2.0|     18.0|       4.0|   3.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|        0.75|          3.0|     16.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|22.0|         1.5|          2.0|     16.0|       5.0|   5.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|27.0|        10.0|          2.0|     14.0|       1.0|   5.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+-------------+-------------+ 
only showing top 20 rows 
  
   
val encodeDF: DataFrame = encoded1
encodeDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 11 more fields] 
   
encodeDF.show() 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+-------------+-------------+ 
|affairs| age|yearsmarried|religiousness|education|occupation|rating|gender|children|genderIndex|    genderVec|childrenIndex|  childrenVec| 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+-------------+-------------+ 
|    0.0|37.0|        10.0|          3.0|     18.0|       7.0|   4.0|  male|      no|        1.0|(2,[1],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     14.0|       6.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|32.0|        15.0|          1.0|     12.0|       1.0|   4.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|57.0|        15.0|          5.0|     18.0|       6.0|   5.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|        0.75|          2.0|     17.0|       6.0|   3.0|  male|      no|        1.0|(2,[1],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|32.0|         1.5|          2.0|     17.0|       5.0|   5.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|22.0|        0.75|          2.0|     12.0|       1.0|   3.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|57.0|        15.0|          2.0|     14.0|       4.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|32.0|        15.0|          4.0|     16.0|       1.0|   2.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|         1.5|          4.0|     14.0|       4.0|   5.0|  male|      no|        1.0|(2,[1],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|37.0|        15.0|          2.0|     20.0|       7.0|   2.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     18.0|       6.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|47.0|        15.0|          5.0|     17.0|       6.0|   4.0|  male|     yes|        1.0|(2,[1],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|         1.5|          2.0|     17.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|27.0|         4.0|          4.0|     14.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|37.0|        15.0|          1.0|     17.0|       5.0|   5.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|37.0|        15.0|          2.0|     18.0|       4.0|   3.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
|    0.0|22.0|        0.75|          3.0|     16.0|       5.0|   4.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|22.0|         1.5|          2.0|     16.0|       5.0|   5.0|female|      no|        0.0|(2,[0],[1.0])|          1.0|(2,[1],[1.0])| 
|    0.0|27.0|        10.0|          2.0|     14.0|       1.0|   5.0|female|     yes|        0.0|(2,[0],[1.0])|          0.0|(2,[0],[1.0])| 
+-------+----+------------+-------------+---------+----------+------+------+--------+-----------+-------------+-------------+-------------+ 
only showing top 20 rows 
   
   
encodeDF.printSchema() 
root 
 |-- affairs: double (nullable = true) 
 |-- age: double (nullable = true) 
 |-- yearsmarried: double (nullable = true) 
 |-- religiousness: double (nullable = true) 
 |-- education: double (nullable = true) 
 |-- occupation: double (nullable = true) 
 |-- rating: double (nullable = true) 
 |-- gender: string (nullable = true) 
 |-- children: string (nullable = true) 
 |-- genderIndex: double (nullable = true) 
 |-- genderVec: vector (nullable = true) 
 |-- childrenIndex: double (nullable = true) 
 |-- childrenVec: vector (nullable = true) 

4.将字段组合成向量feature

//将字段组合成向量feature 
val assembler = new VectorAssembler().setInputCols(Array("affairs", "age", "yearsmarried", "religiousness", "education", "occupation", "rating", "genderVec", "childrenVec")).setOutputCol("features") 
assembler: org.apache.spark.ml.feature.VectorAssembler = vecAssembler_df76d5d1e3f4
   
val vecDF: DataFrame = assembler.transform(encodeDF) 
vecDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 12 more fields] 
   
vecDF.select("features").show 
+--------------------+ 
|            features| 
+--------------------+ 
|[0.0,37.0,10.0,3....| 
|[0.0,27.0,4.0,4.0...| 
|[0.0,32.0,15.0,1....| 
|[0.0,57.0,15.0,5....| 
|[0.0,22.0,0.75,2....| 
|[0.0,32.0,1.5,2.0...| 
|[0.0,22.0,0.75,2....| 
|[0.0,57.0,15.0,2....| 
|[0.0,32.0,15.0,4....| 
|[0.0,22.0,1.5,4.0...| 
|[0.0,37.0,15.0,2....| 
|[0.0,27.0,4.0,4.0...| 
|[0.0,47.0,15.0,5....| 
|[0.0,22.0,1.5,2.0...| 
|[0.0,27.0,4.0,4.0...| 
|[0.0,37.0,15.0,1....| 
|[0.0,37.0,15.0,2....| 
|[0.0,22.0,0.75,3....| 
|[0.0,22.0,1.5,2.0...| 
|[0.0,27.0,10.0,2....| 
+--------------------+ 
only showing top 20 rows 

5.标准化--均值标准差

// 标准化--均值标准差 
val scaler = new StandardScaler().setInputCol("features").setOutputCol("scaledFeatures").setWithStd(true).setWithMean(true) 
scaler: org.apache.spark.ml.feature.StandardScaler = stdScal_43d3da1cd3bf 
   
// Compute summary statistics by fitting the StandardScaler. 
val scalerModel = scaler.fit(vecDF) 
scalerModel: org.apache.spark.ml.feature.StandardScalerModel = stdScal_43d3da1cd3bf 
   
// Normalize each feature to have unit standard deviation. 
val scaledData: DataFrame = scalerModel.transform(vecDF) 
scaledData: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 13 more fields] 
   
scaledData.select("features", "scaledFeatures").show 
+--------------------+--------------------+ 
|            features|      scaledFeatures| 
+--------------------+--------------------+ 
|[0.0,37.0,10.0,3....|[-0.4413500298573...| 
|[0.0,27.0,4.0,4.0...|[-0.4413500298573...| 
|[0.0,32.0,15.0,1....|[-0.4413500298573...| 
|[0.0,57.0,15.0,5....|[-0.4413500298573...| 
|[0.0,22.0,0.75,2....|[-0.4413500298573...| 
|[0.0,32.0,1.5,2.0...|[-0.4413500298573...| 
|[0.0,22.0,0.75,2....|[-0.4413500298573...| 
|[0.0,57.0,15.0,2....|[-0.4413500298573...| 
|[0.0,32.0,15.0,4....|[-0.4413500298573...| 
|[0.0,22.0,1.5,4.0...|[-0.4413500298573...| 
|[0.0,37.0,15.0,2....|[-0.4413500298573...| 
|[0.0,27.0,4.0,4.0...|[-0.4413500298573...| 
|[0.0,47.0,15.0,5....|[-0.4413500298573...| 
|[0.0,22.0,1.5,2.0...|[-0.4413500298573...| 
|[0.0,27.0,4.0,4.0...|[-0.4413500298573...| 
|[0.0,37.0,15.0,1....|[-0.4413500298573...| 
|[0.0,37.0,15.0,2....|[-0.4413500298573...| 
|[0.0,22.0,0.75,3....|[-0.4413500298573...| 
|[0.0,22.0,1.5,2.0...|[-0.4413500298573...| 
|[0.0,27.0,10.0,2....|[-0.4413500298573...| 
+--------------------+--------------------+ 
only showing top 20 rows 

6.主成分PCA

// 主成分 
val pca = new PCA().setInputCol("scaledFeatures").setOutputCol("pcaFeatures").setK(3).fit(scaledData) 
   
pca.explainedVariance.values //解释变量方差 
res11: Array[Double] = Array(0.28779526464781313, 0.23798543640278289, 0.11742828783633019) 
   
pca.pc //载荷（观测变量与主成分的相关系数） 
res12: org.apache.spark.ml.linalg.DenseMatrix =
-0.12034310848156521  0.05153952289637974   0.6678769450480689
-0.42860623714516627  0.05417889891307473   -0.05592377098140197
-0.44404074412877986  0.1926596811059294    -0.017025575192258197
-0.12233707317255231  0.08053139375662526   -0.5093149296300096
-0.14664751606128462  -0.3872166556211308   -0.03406819489501708
-0.145543746024348    -0.43054860653839705  0.07841454709046872
0.17703994181974803   -0.12792784984216296  -0.5173229755329072
0.2459668445061567    0.4915809641798787    0.010477548320795945
-0.2459668445061567   -0.4915809641798787   -0.010477548320795945
-0.44420980045271047  0.240652448514566     -0.089356723885704
0.4442098004527103    -0.24065244851456588  0.08935672388570405
   
pca.extractParamMap() 
res13: org.apache.spark.ml.param.ParamMap =
{ 
    pca_40a453a54776-inputCol: scaledFeatures, 
    pca_40a453a54776-k: 3, 
    pca_40a453a54776-outputCol: pcaFeatures 
} 
   
pca.params 
res14: Array[org.apache.spark.ml.param.Param[_]] = Array(pca_40a453a54776__inputCol, pca_40a453a54776__k, pca_40a453a54776__outputCol) 
   
  
   
val pcaDF: DataFrame = pca.transform(scaledData) 
pcaDF: org.apache.spark.sql.DataFrame = [affairs: double, age: double ... 14 more fields] 
   
pcaDF.cache() 
res15: pcaDF.type = [affairs: double, age: double ... 14 more fields] 
   
   
pcaDF.printSchema() 
root 
 |-- affairs: double (nullable = true) 
 |-- age: double (nullable = true) 
 |-- yearsmarried: double (nullable = true) 
 |-- religiousness: double (nullable = true) 
 |-- education: double (nullable = true) 
 |-- occupation: double (nullable = true) 
 |-- rating: double (nullable = true) 
 |-- gender: string (nullable = true) 
 |-- children: string (nullable = true) 
 |-- genderIndex: double (nullable = true) 
 |-- genderVec: vector (nullable = true) 
 |-- childrenIndex: double (nullable = true) 
 |-- childrenVec: vector (nullable = true) 
 |-- features: vector (nullable = true) 
 |-- scaledFeatures: vector (nullable = true) 
 |-- pcaFeatures: vector (nullable = true) 
   
   
pcaDF.select("features", "scaledFeatures", "pcaFeatures").show 
+--------------------+--------------------+--------------------+ 
|            features|      scaledFeatures|         pcaFeatures| 
+--------------------+--------------------+--------------------+ 
|[0.0,37.0,10.0,3....|[-0.4413500298573...|[0.27828160409293...| 
|[0.0,27.0,4.0,4.0...|[-0.4413500298573...|[2.42147114101165...| 
|[0.0,32.0,15.0,1....|[-0.4413500298573...|[0.18301418047489...| 
|[0.0,57.0,15.0,5....|[-0.4413500298573...|[-2.9795960667914...| 
|[0.0,22.0,0.75,2....|[-0.4413500298573...|[1.79299133565688...| 
|[0.0,32.0,1.5,2.0...|[-0.4413500298573...|[2.65694237441759...| 
|[0.0,22.0,0.75,2....|[-0.4413500298573...|[3.48234503794570...| 
|[0.0,57.0,15.0,2....|[-0.4413500298573...|[-2.4215838062079...| 
|[0.0,32.0,15.0,4....|[-0.4413500298573...|[-0.6964555195741...| 
|[0.0,22.0,1.5,4.0...|[-0.4413500298573...|[2.18771069800414...| 
|[0.0,37.0,15.0,2....|[-0.4413500298573...|[-2.4259075891377...| 
|[0.0,27.0,4.0,4.0...|[-0.4413500298573...|[-0.7743038356008...| 
|[0.0,47.0,15.0,5....|[-0.4413500298573...|[-2.6176149267534...| 
|[0.0,22.0,1.5,2.0...|[-0.4413500298573...|[2.95788535193022...| 
|[0.0,27.0,4.0,4.0...|[-0.4413500298573...|[2.50146472861263...| 
|[0.0,37.0,15.0,1....|[-0.4413500298573...|[-0.5123817022008...| 
|[0.0,37.0,15.0,2....|[-0.4413500298573...|[-0.9191740114044...| 
|[0.0,22.0,0.75,3....|[-0.4413500298573...|[2.97391491782863...| 
|[0.0,22.0,1.5,2.0...|[-0.4413500298573...|[3.17940505267806...| 
|[0.0,27.0,10.0,2....|[-0.4413500298573...|[0.74585406839527...| 
+--------------------+--------------------+--------------------+ 
only showing top 20 rows 

7.聚类

// 注意最大迭代次數和轮廓系数 
   
val KSSE = (2 to 20 by 1).toList.map { k =>
      // 聚类
      // Trains a k-means model.
      val kmeans = new KMeans().setK(k).setSeed(1L).setFeaturesCol("scaledFeatures")
      val model = kmeans.fit(scaledData)

      // Evaluate clustering by computing Within Set Sum of Squared Errors.
      val WSSSE = model.computeCost(scaledData)

      // K，实际迭代次数，SSE，聚类类别编号，每类的记录数，类中心点
      (k, model.getMaxIter, WSSSE, model.summary.cluster, model.summary.clusterSizes, model.clusterCenters)
    }

    // 根据SSE确定K值
    val KSSEdf:DataFrame=KSSE.map{x=>(x._1,x._2,x._3,x._5)}.toDF("K", "MaxIter", "SSE", "clusterSizes")
   
KSSE.foreach(println)