Spark 二项逻辑回归__二分类

Posted soyosuyang

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Spark 二项逻辑回归__二分类相关的知识，希望对你有一定的参考价值。

package Spark_MLlib

import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.classification.{BinaryLogisticRegressionSummary, LogisticRegression, LogisticRegressionModel}
import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator
import org.apache.spark.ml.feature.{IndexToString, StringIndexer, VectorIndexer}
import org.apache.spark.sql.SparkSession
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.sql.functions

case class data_schema(features:Vector,label:String)
object 二项逻辑回归__二分类 {
  val spark=SparkSession.builder().master("local").getOrCreate()
  import spark.implicits._  //支持把一个RDD隐式转换为一个DataFrame
  def main(args: Array[String]): Unit = {
    val df =spark.sparkContext.textFile("file:///home/soyo/桌面/spark编程测试数据/soyo.txt")
      .map(_.split(",")).map(x=>data_schema(Vectors.dense(x(0).toDouble,x(1).toDouble,x(2).toDouble,x(3).toDouble),x(4))).toDF()
      df.show(130)
      df.createOrReplaceTempView("data_schema")
     val df_data=spark.sql("select * from data_schema where label !=‘soyo2‘") //这里soyo2需要加单引号,不然报错
     // df_data.map(x=>x(1)+":"+x(0)).collect().foreach(println)
        df_data.show()
     val labelIndexer=new StringIndexer().setInputCol("label").setOutputCol("indexedLabel").fit(df_data)
     val featureIndexer=new VectorIndexer().setInputCol("features").setOutputCol("indexedFeatures").fit(df_data)  //目的在特征向量中建类别索引
     val Array(trainData,testData)=df_data.randomSplit(Array(0.7,0.3))
     val lr=new LogisticRegression().setLabelCol("indexedLabel").setFeaturesCol("indexedFeatures").setMaxIter(10).setRegParam(0.5).setElasticNetParam(0.8)//setRegParam:正则化参数,设置elasticnet混合参数为0.8,setFamily("multinomial"):设置为多项逻辑回归,不设置setFamily为二项逻辑回归
     val labelConverter=new IndexToString().setInputCol("prediction").setOutputCol("predictionLabel").setLabels(labelIndexer.labels)

     val lrPipeline=new Pipeline().setStages(Array(labelIndexer,featureIndexer,lr,labelConverter))
     val lrPipeline_Model=lrPipeline.fit(trainData)
     val lrPrediction=lrPipeline_Model.transform(testData)
    lrPrediction.show(false)
    // lrPrediction.take(100).foreach(println)
     //模型评估
    val evaluator=new MulticlassClassificationEvaluator().setLabelCol("indexedLabel").setPredictionCol("prediction")
    val lrAccuracy=evaluator.evaluate(lrPrediction)
     println("准确率为： "+lrAccuracy)
    val lrError=1-lrAccuracy
    println("错误率为: "+lrError)
    val LRmodel=lrPipeline_Model.stages(2).asInstanceOf[LogisticRegressionModel]
    println("二项逻辑回归模型系数的向量: "+LRmodel.coefficients)
    println("二项逻辑回归模型的截距: "+LRmodel.intercept)
    println("类的数量(标签可以使用的值): "+LRmodel.numClasses)
    println("模型所接受的特征的数量: "+LRmodel.numFeatures)
    //对模型的总结(summary)目前只支持二项逻辑斯蒂回归,多项式逻辑回归并不支持(用的是spark 2.2.0)
   println(LRmodel.hasSummary)
    val trainingSummary = LRmodel.summary
    //损失函数,可以看到损失函数随着循环是逐渐变小的,损失函数越小,模型就越好
    val objectiveHistory =trainingSummary.objectiveHistory
    objectiveHistory.foreach(println)
    //强制转换为BinaryLogisticRegressionSummary
    val binarySummary= trainingSummary.asInstanceOf[BinaryLogisticRegressionSummary]
    //ROC曲线下方的面积,越接近1说明模型越好
    val area_ROC=binarySummary.areaUnderROC
    println("ROC 曲线下的面积为: "+area_ROC)
    //fMeasureByThreshold:返回一个带有beta = 1.0的两个字段(阈值,f - measure)曲线的dataframe
     val fMeasure=binarySummary.fMeasureByThreshold
    println("fMeasure的行数: "+fMeasure.collect().length)
    fMeasure.show(100)
    val maxFMeasure=fMeasure.select(functions.max("F-Measure")).head().getDouble(0)
    println("最大的F-Measure的值为: "+maxFMeasure)
    //最优的阀值
    val bestThreashold=fMeasure.where($"F-Measure"===maxFMeasure).select("threshold").head().getDouble(0)
    println("最优的阀值为："+bestThreashold)
    /* 这样求的不是最优的阀值
     val s=fMeasure.select(functions.max("threshold")).head().getDouble(0)
    println(s)
    */
    LRmodel.setThreshold(bestThreashold)

  }
}

结果：


+-----------------+-----+------------+------------------+--------------------------------------------+----------------------------------------+----------+---------------+
|features         |label|indexedLabel|indexedFeatures   |rawPrediction                               |probability                             |prediction|predictionLabel|
+-----------------+-----+------------+------------------+--------------------------------------------+----------------------------------------+----------+---------------+
|[4.4,2.9,1.4,0.2]|soyo1|0.0         |[4.4,2.9,1.4,1.0] |[0.0690256519103008,-0.0690256519103008]    |[0.5172495646670774,0.48275043533292256]|0.0       |soyo1          |
|[4.4,3.0,1.3,0.2]|soyo1|0.0         |[4.4,3.0,1.3,1.0] |[0.07401171769156373,-0.07401171769156373]  |[0.518494487869238,0.481505512130762]   |0.0       |soyo1          |
|[4.6,3.1,1.5,0.2]|soyo1|0.0         |[4.6,3.1,1.5,1.0] |[0.06403958612903785,-0.06403958612903785]  |[0.5160044273015656,0.48399557269843435]|0.0       |soyo1          |
|[4.6,3.2,1.4,0.2]|soyo1|0.0         |[4.6,3.2,1.4,1.0] |[0.0690256519103008,-0.0690256519103008]    |[0.5172495646670774,0.48275043533292256]|0.0       |soyo1          |
|[4.6,3.6,1.0,0.2]|soyo1|0.0         |[4.6,3.6,1.0,1.0] |[0.08896991503535255,-0.08896991503535255]  |[0.5222278183980882,0.4777721816019118] |0.0       |soyo1          |
|[4.8,3.0,1.4,0.1]|soyo1|0.0         |[4.8,3.0,1.4,0.0] |[0.0690256519103008,-0.0690256519103008]    |[0.5172495646670774,0.48275043533292256]|0.0       |soyo1          |
|[4.9,2.5,4.5,1.7]|soyo3|1.0         |[4.9,2.5,4.5,9.0] |[-0.08554238730885033,0.08554238730885033]  |[0.47862743439605193,0.5213725656039481]|1.0       |soyo3          |
|[5.0,3.0,1.6,0.2]|soyo1|0.0         |[5.0,3.0,1.6,1.0] |[0.059053520347774904,-0.059053520347774904]|[0.5147590911988562,0.48524090880114373]|0.0       |soyo1          |
|[5.1,3.5,1.4,0.3]|soyo1|0.0         |[5.1,3.5,1.4,2.0] |[0.0690256519103008,-0.0690256519103008]    |[0.5172495646670774,0.48275043533292256]|0.0       |soyo1          |
|[5.1,3.8,1.6,0.2]|soyo1|0.0         |[5.1,3.8,1.6,1.0] |[0.059053520347774904,-0.059053520347774904]|[0.5147590911988562,0.48524090880114373]|0.0       |soyo1          |
|[5.3,3.7,1.5,0.2]|soyo1|0.0         |[5.3,3.7,1.5,1.0] |[0.06403958612903785,-0.06403958612903785]  |[0.5160044273015656,0.48399557269843435]|0.0       |soyo1          |
|[5.4,3.7,1.5,0.2]|soyo1|0.0         |[5.4,3.7,1.5,1.0] |[0.06403958612903785,-0.06403958612903785]  |[0.5160044273015656,0.48399557269843435]|0.0       |soyo1          |
|[5.4,3.9,1.7,0.4]|soyo1|0.0         |[5.4,3.9,1.7,3.0] |[0.05406745456651198,-0.05406745456651198]  |[0.5135135717949689,0.486486428205031]  |0.0       |soyo1          |
|[5.7,3.8,1.7,0.3]|soyo1|0.0         |[5.7,3.8,1.7,2.0] |[0.05406745456651198,-0.05406745456651198]  |[0.5135135717949689,0.486486428205031]  |0.0       |soyo1          |
|[5.8,2.8,5.1,2.4]|soyo3|1.0         |[5.8,2.8,5.1,16.0]|[-0.11545878199642795,0.11545878199642795]  |[0.4711673274353307,0.5288326725646694] |1.0       |soyo3          |
|[5.8,4.0,1.2,0.2]|soyo1|0.0         |[5.8,4.0,1.2,1.0] |[0.07899778347282668,-0.07899778347282668]  |[0.5197391814925231,0.480260818507477]  |0.0       |soyo1          |
|[6.1,3.0,4.9,1.8]|soyo3|1.0         |[6.1,3.0,4.9,10.0]|[-0.10548665043390212,0.10548665043390212]  |[0.4736527642876721,0.5263472357123279] |1.0       |soyo3          |
|[6.3,2.7,4.9,1.8]|soyo3|1.0         |[6.3,2.7,4.9,10.0]|[-0.10548665043390212,0.10548665043390212]  |[0.4736527642876721,0.5263472357123279] |1.0       |soyo3          |
|[6.3,2.9,5.6,1.8]|soyo3|1.0         |[6.3,2.9,5.6,10.0]|[-0.14038911090274264,0.14038911090274264]  |[0.46496025354157383,0.5350397464584261]|1.0       |soyo3          |
|[6.5,3.0,5.5,1.8]|soyo3|1.0         |[6.5,3.0,5.5,10.0]|[-0.13540304512147971,0.13540304512147971]  |[0.4662008623530858,0.5337991376469143] |1.0       |soyo3          |
+-----------------+-----+------------+------------------+--------------------------------------------+----------------------------------------+----------+---------------+
only showing top 20 rows

准确率为： 1.0
错误率为: 0.0
二项逻辑回归模型系数的向量: [0.0,0.0,0.0498606578126294,-0.0]
二项逻辑回归模型的截距: -0.13883057284798195
类的数量(标签可以使用的值): 2
模型所接受的特征的数量: 4
true
0.6927819059876479
0.6921535505946383
0.6902127176671448
0.6898394130469451
0.689535794969328
0.6894009255584304
0.6893497986701255
0.689265433291139
0.6887228224555286
0.6895877386375889
0.6872109190567809
ROC 曲线下的面积为: 1.0
fMeasure的行数: 26
+-------------------+-------------------+
|          threshold|          F-Measure|
+-------------------+-------------------+
| 0.5511227178429281|0.05128205128205127|
| 0.5486545095952616|                0.1|
|  0.547419499422364|0.14634146341463414|
| 0.5449477416103359| 0.1904761904761905|
| 0.5412359859690851| 0.2727272727272727|
| 0.5399976958289747|0.34782608695652173|
| 0.5387589116841329|0.38297872340425526|
| 0.5375196486465557| 0.4799999999999999|
| 0.5362799218518347| 0.5098039215686275|
| 0.5350397464584261| 0.6428571428571429|
| 0.5337991376469143| 0.6896551724137931|
| 0.5325581106192748| 0.7333333333333334|
| 0.5313166805981351| 0.7741935483870968|
| 0.5300748628260323| 0.8125000000000001|
| 0.5288326725646694| 0.9142857142857143|
| 0.5275901250941695|  0.958904109589041|
| 0.5263472357123279|  0.972972972972973|
| 0.5251040197338624|                1.0|
| 0.4889779551275146| 0.9743589743589743|
|  0.486486428205031| 0.9500000000000001|
|0.48524090880114373| 0.8941176470588235|
|0.48399557269843435| 0.7916666666666666|
|0.48275043533292256| 0.7307692307692308|
|  0.481505512130762| 0.6909090909090909|
|  0.480260818507477| 0.6846846846846847|
|0.47901636986720014| 0.6785714285714285|
+-------------------+-------------------+

最大的F-Measure的值为: 1.0
最优的阀值为：0.5251040197338624

以上是关于Spark 二项逻辑回归__二分类的主要内容，如果未能解决你的问题，请参考以下文章

用二项逻辑斯蒂回归解决二分类问题

机器学习算法应用和学习_2_理论篇2.2 M_分类_逻辑回归

Spark 多项式逻辑回归__多分类

机器学习之逻辑回归以及梯度下降法求解

逻辑回归算法实现_基于R语言

逻辑回归-4.添加多项式特征