机器学习之线性回归算法

线性回归：通过历史数据拟合出一条直线，用这条直线对新的数据进行预测。

房价预测分析：

package cn.doitedu.ml
import org.apache.log4j.{Level, Logger}
import org.apache.spark.ml.evaluation.RegressionEvaluator
import org.apache.spark.ml.linalg
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.ml.regression.LinearRegression
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.expressions.UserDefinedFunction
import scala.collection.mutable
object LinearPrice {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache").setLevel(Level.WARN)
val spark = SparkSession
.builder()
.appName("房价预测")
.master("local")
.getOrCreate()
import org.apache.spark.sql.functions._
import spark.implicits._
val df = spark.read.option("header",true).option("inferSchema",true).csv("userprofile/data/linear/sample")
val arr2Vec: UserDefinedFunction = udf((arr:mutable.WrappedArray[Double])=>{
// Vector是一个接口，它有两个实现，一个是DenseVector，一个是SparseVector
val vector: linalg.Vector = Vectors.dense(arr.toArray)
vector
})
// area,floor,price
val vecDF = df.select(arr2Vec(array('area,'floor)) as "features",'price )
// 构造算法对象
val linearRegression = new LinearRegression()
.setRegParam(0.1)   // 正则化参数  防止过拟合
.setLabelCol("price")
.setFeaturesCol("features")
// 训练模型
val model = linearRegression.fit(vecDF)
// 加载测试数据
val test = spark.read.option("header",true).option("inferSchema",true).csv("userprofile/data/linear/test")
val testVecDF = df.select(arr2Vec(array('area,'floor)) as "features",'price )
// 用训练好的模型，来对测试数据进行输出的预测
val result = model.transform(testVecDF)
result.show(100,false)
// 评估预测效果
val regressionEvaluator = new RegressionEvaluator()
.setPredictionCol("prediction")
.setLabelCol("price")
.setMetricName("rmse")
val rmse: Double = regressionEvaluator.evaluate(result)
spark.close()
}
}