机器学习技法：Homework #5 特征变换&Soft-Margin SVM相关习题

Posted 2021-11-26 cherrychenlee

原文地址：https://www.jianshu.com/p/6bf801bdc644

特征变换

问题描述

程序实现

# coding: utf-8

import numpy as np
from cvxopt import matrix, solvers
from sklearn import svm


def gen_data():
    X = [[1, 0], [0, 1], [0, -1], [-1, 0], [0, 2], [0, -2], [-2, 0]]
    X = np.array(X)
    y = [-1, -1, -1, 1, 1, 1, 1]
    y = np.array(y)
    assert X.shape[0] == y.shape[0] and X.shape[1] == 2, "wrong data shape!"
    return X, y


def explict_transform(X):
    assert X.shape[1] == 2, "wrong shape of X!"
    num = X.shape[0]
    X1 = X[:, 0]
    X2 = X[:, 1]
    new_X1 = X2 ** 2 - 2 * X1 + 3
    new_X2 = X1 ** 2 - 2 * X2 - 3
    new_X = np.concatenate((new_X1.reshape((num, 1)), new_X2.reshape(num, 1)), axis=1)
    return new_X


def svm_hard_linear(X, y):
    num, dim = X.shape
    P = matrix(np.concatenate((np.zeros((1, 1 + dim)),
                               np.concatenate((np.zeros((dim, 1)), np.eye(dim)), axis=1)), axis=0), tc='d')
    q = matrix(np.zeros((1 + dim, 1)), tc='d')
    G = matrix(-y * np.concatenate((np.ones((num, 1), dtype=np.float), X), axis=1), tc='d')
    h = matrix(-np.ones((num, 1)), tc='d')
    sol = solvers.qp(P, q, G, h)
    return sol['x']


def implicit_transform(X):
    assert X.shape[1] == 2, "wrong shape of X!"
    num=X.shape[0]
    X1 = X[:, 0]
    X2 = X[:, 1]
    new_X1=np.ones((num,1))
    new_X2=2**(0.5)*X1
    new_X3=2**(0.5)*X2
    new_X4=X1**2
    new_X5=X2**2
    new_X6=2**(0.5)*X1*X2
    new_X = np.concatenate((new_X1.reshape((num, 1)), new_X2.reshape(num, 1),new_X3.reshape(num, 1),
                            new_X4.reshape(num, 1),new_X5.reshape(num, 1),new_X6.reshape(num, 1)), axis=1)
    return new_X

if __name__ == "__main__":
    np.set_printoptions(precision=6,suppress=True)
    X, y = gen_data()

    # explicit
    # 2
    exp_X= explict_transform(X)
    u = np.array(svm_hard_linear(exp_X, y.reshape(y.shape[0],1)))
    b = u[0, :]
    w = u[1:, :]
    print("b:\n", b)
    print("w:\n", w)

    # implicit
    clf=svm.SVC(C=1000000,kernel='poly',degree=2,gamma=1,coef0=1)
    clf.fit(X,y)
    # 3
    alpha_y=clf.dual_coef_
    alpha_y=alpha_y.reshape((alpha_y.shape[1],))
    sv_ID=clf.support_
    sv_y=[]
    for i in range(sv_ID.shape[0]):
        sv_y.append(y[sv_ID[i]])
    alpha=[alpha_y[i]/sv_y[i] for i in range(sv_ID.shape[0])]
    print("alpha*y:\n",alpha_y)
    print("alpha:\n",alpha)
    sv_X=clf.support_vectors_
    print("support vectors:\n",sv_X)
    # 4
    b=clf.intercept_
    print("b:\n",b)
    w=np.dot(alpha_y,implicit_transform(sv_X)).reshape((6,1))
    print("w:\n",w)

运行结果

Soft-Margin SVM

问题描述

程序实现

# coding: utf-8


import numpy as np
from sklearn import svm
import matplotlib.pyplot as plt


def read_data(dataFile):
    with open(dataFile,'r') as f:
        lines=f.readlines()
        data_list=[]
        for line in lines:
            line=line.strip().split()
            data_list.append([float(l) for l in line])
        dataArray=np.array(data_list)
        num_data=dataArray.shape[0]
        num_dim=dataArray.shape[1]-1
        dataX=dataArray[:,1:].reshape((num_data,num_dim))
        dataY=dataArray[:,0].reshape((num_data,))
        return dataX,dataY

data_X,data_Y=read_data("features.train")
test_X,test_Y=read_data("features.test")


def convert_label(dataY,chosen_class):
    num=dataY.shape[0]
    new_Y=-np.ones_like(dataY)
    for i in range(num):
        if dataY[i]==chosen_class:
            new_Y[i]=1
    return new_Y


def zero_one_cost(pred,Y):
    assert pred.shape==Y.shape,"wrong shape of pred and Y!"
    return np.sum(np.not_equal(pred,Y))/Y.shape[0]


def question15():
    c_list=[-6,-4,-2,0,2]
    w_list=[]
    new_Y=convert_label(data_Y,0)
    for i in c_list:
        clf=svm.LinearSVC(loss="hinge",C=10**i)
        clf.fit(data_X,new_Y)
        w_list.append(np.sqrt(np.sum(clf.coef_**2)))
    plt.figure(figsize=(10,6))
    plt.plot(c_list,w_list,'b')
    plt.plot(c_list,w_list,'ro')
    for (c,w) in zip(c_list,w_list):
        plt.text(c+0.1,w,str(round(w,4)))
    plt.xlabel("log10(C)")
    plt.ylabel("||w||")
    plt.xlim(-8,4)
    plt.title("||w|| versus log10(C)")
    plt.savefig("15.png")


def question16and17():
    # 16
    c_list = [-6, -4, -2, 0, 2]
    Ein_list=[]
    alpha_sum_list=[]
    new_Y=convert_label(data_Y,8)
    for i in c_list:
        clf=svm.SVC(C=10**i,kernel='poly',degree=2,gamma=1,coef0=1)
        clf.fit(data_X,new_Y)
        pred=clf.predict(data_X)
        Ein_list.append(zero_one_cost(pred,new_Y))
        alpha_sum_list.append(np.sum(np.abs(clf.dual_coef_)))
        # print(np.sum(clf.dual_coef_))
        # print(clf.n_support_)
    plt.figure(figsize=(10,6))
    plt.plot(c_list,Ein_list,'b')
    plt.plot(c_list,Ein_list,'ro')
    for (c,e) in zip(c_list,Ein_list):
        plt.text(c+0.1,e,str(round(e,4)))
    plt.xlabel("log10(C)")
    plt.ylabel("Ein")
    plt.xlim(-8, 4)
    plt.title("Ein versus log10(C)")
    plt.savefig("16.png")
    # 17
    plt.figure(figsize=(10,6))
    plt.plot(c_list,alpha_sum_list,'b')
    plt.plot(c_list,alpha_sum_list,'ro')
    for (c,a) in zip(c_list,alpha_sum_list):
        plt.text(c+0.1,a,str(round(a,6)))
    plt.xlabel("log10(C)")
    plt.ylabel("sum of alpha")
    plt.xlim(-8, 4)
    plt.title("sum of alpha versus log10(C)")
    plt.savefig("17.png")


def question18():
    c_list=[-3,-2,-1,0,1]
    dis_list=[]
    new_Y=convert_label(data_Y,0)
    for i in c_list:
        clf=svm.SVC(C=10**i,kernel='rbf',gamma=100)
        clf.fit(data_X,new_Y)
        sv_ID=clf.support_
        dis_list.append(new_Y[sv_ID[0]]*clf.decision_function(data_X)[sv_ID[0]])
    plt.figure(figsize=(10,6))
    plt.plot(c_list,dis_list,'b')
    plt.plot(c_list,dis_list,'ro')
    for (c,w) in zip(c_list,dis_list):
        plt.text(c+0.1,w,str(round(w,4)))
    plt.xlabel("log10(C)")
    plt.ylabel("free sv's function distance to hyperplane")
    plt.xlim(-5, 3)
    plt.ylim(ymax=1.01)
    plt.title("free sv's function distance to hyperplane versus log10(C)")
    plt.savefig("18.png")


def question19():
    new_Y=convert_label(data_Y,0)
    new_test_Y=convert_label(test_Y,0)
    gamma_list=[0,1,2,3,4]
    Eout_list=[]
    for i in gamma_list:
        clf=svm.SVC(C=0.1,kernel='rbf',gamma=10**i)
        clf.fit(data_X,new_Y)
        pred=clf.predict(test_X)
        Eout_list.append(zero_one_cost(pred,new_test_Y))
    plt.figure(figsize=(10,6))
    plt.plot(gamma_list,Eout_list,'b')
    plt.plot(gamma_list,Eout_list,'ro')
    for (c,w) in zip(gamma_list,Eout_list):
        plt.text(c+0.1,w,str(round(w,4)))
    plt.xlabel("log10(gamma)")
    plt.ylabel("Eout")
    plt.xlim(-1, 5)
    plt.ylim(ymax=0.19)
    plt.title("Eout versus log10(C)")
    plt.savefig("19.png")


def question20():
    new_Y=convert_label(data_Y,0)
    gamma_list=[0,1,2,3,4]
    chosen_gamma=[]
    for t in range(100):
        np.random.seed(t)
        chosenID=np.random.randint(0,data_X.shape[0],1000)
        train_X=[]
        train_Y=[]
        val_X=[]
        val_Y=[]
        for i in range(data_X.shape[0]):
            if(i not in chosenID):
                train_X.append(data_X[i,:])
                train_Y.append(new_Y[i])
            else:
                val_X.append(data_X[i,:])
                val_Y.append(new_Y[i])
        train_X=np.array(train_X)
        train_Y=np.array(train_Y)
        val_X=np.array(val_X)
        val_Y=np.array(val_Y)
        Eval_list=[]
        for g in gamma_list:
            clf=svm.SVC(C=0.1,kernel='rbf',gamma=10**g)
            clf.fit(train_X,train_Y)
            pred=clf.predict(val_X)
            Eval_list.append(zero_one_cost(pred,val_Y))
        chosen_gamma.append(gamma_list[Eval_list.index(min(Eval_list))])
    times=[]
    for i in gamma_list:
        times.append(chosen_gamma.count(i))
    plt.figure(figsize=(10,6))
    plt.bar(left=(gamma_list),height=(times),width=1,align="center",yerr=0.000001)
    for (c,w) in zip(gamma_list,times):
        plt.text(c,w*1.03,str(round(w,4)))
    plt.xlabel("log10(gamma)")
    plt.ylabel("the number of chosen times")
    plt.xlim(-1, 5)
    plt.ylim(0,80)
    plt.title("the number of chosen times for gamma")
    plt.savefig("20.png")


if __name__=="__main__":

    question15()
    question16and17()
    question18()
    question19()
    question20()

运行结果