梯度下降算法笔记

Posted 2020-08-21 coskaka

　　今天课上主要讲的是梯度下降算法。

　　上一次老师留了梯度下降算法求解线性回归的作业。大部分用java和C++实现的。

　　笔记也主要来自课程ppt，老师课程的ppt也主要参考的斯坦福吴恩达老师的讲义。

　　梯度下降法（英语：Gradient descent）是一个一阶最优化算法，通常也称为最速下降法。 要使用梯度下降法找到一个函数的局部极小值，必须向函数上当前点对应梯度（或者是近似梯度）的反方向的规定步长距离点进行迭代搜索。如果相反地向梯度正方向迭代进行搜索，则会接近函数的局部极大值点；这个过程则被称为梯度上升法。[来自维基百科]

　　

  1 __author__ = \'Oscar_Yang\'
  2 #-*- coding= utf-8 -*-
  3 """
  4 实现线性回归参数预测
  5 """
  6 class GD(object):
  7     def __init__(self, e=0.03,m=5,rate=0.02,maxIterations=10000):#初始化属性
  8         self.e = e
  9         self.m = m
 10         self.rate = rate
 11         self.maxIterations=maxIterations
 12     """
 13     梯度下降拟合函数
 14     """
 15     def fit(self,X,y,data):
 16         self.theta = np.ones(data.shape[1])
 17         self.cost_ = []
 18         for i in range(0, self.maxIterations):
 19             output = self.predict(X)
 20             errors = (y - output)
 21             cost = (errors ** 2).sum() / 2.0
 22             self.cost_.append(cost)
 23             if cost < self.e:
 24                 break
 25             else:
 26                 gradient = np.dot(X.T,errors) / self.m
 27                 self.theta = self.theta + self.rate * gradient
 28         return self
 29     """
 30         normal equation
 31     """
 32     @staticmethod
 33     def fit_equation(X,y):
 34         xTx = X.T.dot(X)
 35         XtX = np.linalg.inv(xTx)
 36         XtX_xT = XtX.dot(X.T)
 37         theta = XtX_xT.dot(y)
 38         return theta
 39 
 40 
 41     def predict(self, X):
 42         return np.dot(X,self.theta)
 43     @classmethod
 44     def cal_time(self):
 45         return time.clock()
 46 if __name__ == \'__main__\':
 47 
 48     import pandas as pd
 49     import numpy as np
 50     import time
 51     import matplotlib.pyplot as plt
 52 
 53     data = pd.read_table("shuju1.txt")
 54     traindata0 = pd.DataFrame(np.ones(data.shape[0]))
 55     traindata1 = data.ix[:, 0:-1]
 56     traindata = pd.concat([traindata0, traindata1], axis=1)
 57     traindata_label = data.ix[:, -1]
 58     """
 59         normal equation
 60     """
 61     real_theta = GD(m=5,rate=0.01).fit_equation(traindata,traindata_label)
 62     print("精确参数：",real_theta)
 63     print("_____________________________________")
 64     fig,ax=plt.subplots(nrows=1,ncols=3,figsize=(18,6))
 65     start_time = GD.cal_time()
 66     ada1 = GD(m=5,rate=0.01).fit(traindata, traindata_label, data=data)
 67     end_time = GD.cal_time()
 68     during_time=end_time-start_time
 69 
 70     print("rate={}时候的参数为：{}".format(ada1.rate,ada1.theta))
 71     print("迭代次数：",len(ada1.cost_))
 72     print("耗时：",during_time)
 73     ax[0].plot(range(1,len(ada1.cost_)+1),np.log10(ada1.cost_),marker="o")
 74     ax[0].set_xlabel("Epochs")
 75     ax[0].set_ylabel(\'log(sum_squ_err)\')
 76     title = "rate:", ada1.rate
 77     ax[0].set_title(title)
 78 
 79     start_time=GD.cal_time()
 80     ada2 = GD(rate=0.03,m=5).fit(traindata, traindata_label, data=data)
 81     end_time = GD.cal_time()
 82     print("_____________________________________")
 83     print("rate={}时候的参数为：{}".format(ada2.rate, ada2.theta))
 84     print("迭代次数：",len(ada2.cost_))
 85     print("耗时：",end_time-start_time)
 86 
 87     ax[1].plot(range(1, len(ada2.cost_)+1), np.log10(ada2.cost_), marker="o")
 88     ax[1].set_xlabel("Epochs")
 89     ax[1].set_ylabel(\'log(sum_squ_err)\')
 90     title="rate:",ada2.rate
 91     ax[1].set_title(title)
 92 
 93 
 94 
 95     start_time=GD.cal_time()
 96     ada3 = GD( rate=0.06, m=5).fit(traindata, traindata_label, data=data)
 97     end_time=GD.cal_time()
 98     # print(ada3.theta)
 99     print("_____________________________________")
100     print("rate={}时候的参数为：{}".format(ada3.rate, ada3.theta))
101     print("迭代次数：",len(ada3.cost_))
102     print("耗时：",end_time-start_time)
103 
104     ax[2].plot(range(1, len(ada3.cost_) + 1), np.log10(ada3.cost_), marker="o")
105     ax[2].set_xlabel("Epochs")
106     ax[2].set_ylabel(\'log(sum_squ_err)\')
107     title = "rate:", ada3.rate
108     ax[2].set_title(title)
109 
110     plt.show()

精确参数： [ 1.75789474 1.21684211 -0.50526316]
_____________________________________
rate=0.01时候的参数为：[ 1.75789374 1.21684043 -0.50526106]
迭代次数： 10000
耗时： 7.593358084384351
_____________________________________
rate=0.03时候的参数为：[ 1.75789474 1.21684211 -0.50526316]
迭代次数： 10000
耗时： 7.8287073625201655
_____________________________________
rate=0.06时候的参数为：[ 1.75789474 1.21684211 -0.50526316]
迭代次数： 10000
耗时： 7.562434044109619