Q-learning简明实例Java代码实现

Posted 2020-10-06 coshaho

在《Q-learning简明实例》中我们介绍了Q-learning算法的简单例子，从中我们可以总结出Q-learning算法的基本思想

本次选择的经验得分 = 本次选择的反馈得分 + 本次选择后场景的历史最佳经验得分

其中反馈得分是单个步骤的价值分值（固定的分值），经验得分是完成目标的学习分值（动态的分值）。

简明实例的Java实现如下

package com.coshaho.learn.qlearning;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;

/**
 * 
 * QLearning.java Create on 2017年9月4日 下午10:08:49    
 *    
 * 类功能说明:   QLearning简明例子实现
 *
 * Copyright: Copyright(c) 2013 
 * Company: COSHAHO
 * @Version 1.0
 * @Author coshaho
 */
public class QLearning 
{
    FeedbackMatrix R = new FeedbackMatrix();
    
    ExperienceMatrix Q = new ExperienceMatrix();
    
    public static void main(String[] args)
    {
        QLearning ql = new QLearning();
        
        for(int i = 0; i < 500; i++)
        {
            Random random = new Random();
            int x = random.nextInt(100) % 6;
            
            System.out.println("第" + i + "次学习, 初始房间是" + x);
            ql.learn(x);
            System.out.println();
        }
    }
    
    public void learn(int x)
    {
        do
        {
            // 随机选择一个联通的房间进入
            int y =  chooseRandomRY(x);
            
            // 获取以进入的房间为起始点的历史最佳得分
            int qy = getMaxQY(y);
            
            // 计算此次移动的得分
            int value = calculateNewQ(x, y, qy);
            Q.set(x, y, value);
            x = y;
        }
        // 走出房间则学习结束
        while(5 != x);
        
        Q.print();
    }
    
    public int chooseRandomRY(int x)
    {
        int[] qRow = R.getRow(x);
        List<Integer> yValues = new ArrayList<Integer>();
        for(int i = 0; i < qRow.length; i++)
        {
            if(qRow[i] >= 0)
            {
                yValues.add(i);
            }
        }

        Random random = new Random();
        int i = random.nextInt(yValues.size()) % yValues.size();
        return yValues.get(i);
    }
    
    public int getMaxQY(int x)
    {
        int[] qRow = Q.getRow(x);
        int length = qRow.length;
        List<YAndValue> yValues = new ArrayList<YAndValue>();
        for(int i = 0; i < length; i++)
        {
            YAndValue yv = new YAndValue(i, qRow[i]);
            yValues.add(yv);
        }
        
        Collections.sort(yValues);
        int num = 1;
        int value = yValues.get(0).getValue();
        for(int i = 1; i < length; i++)
        {
            if(yValues.get(i).getValue() == value)
            {
                num = i + 1;
            }
            else
            {
                break;
            }
        }
        
        Random random = new Random();
        int i = random.nextInt(num) % num;
        return yValues.get(i).getY();
    }
    
    // Q(x,y) = R(x,y) + 0.8 * max(Q(y,i))
    public int calculateNewQ(int x, int y, int qy)
    {
        return (int) (R.get(x, y) + 0.8 * Q.get(y, qy));
    }
    
    public static class YAndValue implements Comparable<YAndValue>
    {
        int y;
        int value;
        
        public int getY() {
            return y;
        }
        public void setY(int y) {
            this.y = y;
        }
        public int getValue() {
            return value;
        }
        public void setValue(int value) {
            this.value = value;
        }
        public YAndValue(int y, int value)
        {
            this.y = y;
            this.value = value;
        }
        public int compareTo(YAndValue o) 
        {
            return o.getValue() - this.value;
        }
    }
}

package com.coshaho.learn.qlearning;

/**
 * 
 * FeedbackMatrix.java Create on 2017年9月4日 下午9:52:41    
 *    
 * 类功能说明:   反馈矩阵
 *
 * Copyright: Copyright(c) 2013 
 * Company: COSHAHO
 * @Version 1.0
 * @Author coshaho
 */
public class FeedbackMatrix 
{
    public int get(int x, int y)
    {
        return R[x][y];
    }
    
    public int[] getRow(int x)
    {
        return R[x];
    }
    
    private static int[][] R = new int[6][6];
    static 
    {
        R[0][0] = -1;
        R[0][1] = -1;
        R[0][2] = -1;
        R[0][3] = -1;
        R[0][4] = 0;
        R[0][5] = -1;
        
        R[1][0] = -1;
        R[1][1] = -1;
        R[1][2] = -1;
        R[1][3] = 0;
        R[1][4] = -1;
        R[1][5] = 100;
        
        R[2][0] = -1;
        R[2][1] = -1;
        R[2][2] = -1;
        R[2][3] = 0;
        R[2][4] = -1;
        R[2][5] = -1;
        
        R[3][0] = -1;
        R[3][1] = 0;
        R[3][2] = 0;
        R[3][3] = -1;
        R[3][4] = 0;
        R[3][5] = -1;
        
        R[4][0] = 0;
        R[4][1] = -1;
        R[4][2] = -1;
        R[4][3] = 0;
        R[4][4] = -1;
        R[4][5] = 100;
        
        R[5][0] = -1;
        R[5][1] = 0;
        R[5][2] = -1;
        R[5][3] = -1;
        R[5][4] = 0;
        R[5][5] = 100;
    }
}

package com.coshaho.learn.qlearning;

/**
 * 
 * ExperienceMatrix.java Create on 2017年9月4日 下午10:03:08    
 *    
 * 类功能说明:   经验矩阵
 *
 * Copyright: Copyright(c) 2013 
 * Company: COSHAHO
 * @Version 1.0
 * @Author coshaho
 */
public class ExperienceMatrix 
{
    public int get(int x, int y)
    {
        return Q[x][y];
    }
    
    public int[] getRow(int x)
    {
        return Q[x];
    }
    
    public void set(int x, int y, int value)
    {
        Q[x][y] = value;
    }
    
    public void print()
    {
        for(int i = 0; i < 6; i++)
        {
            for(int j = 0; j < 6; j++)
            {
                String s = Q[i][j] + "  ";
                if(Q[i][j] < 10)
                {
                    s = s + "  ";
                }
                else if(Q[i][j] < 100)
                {
                    s = s + " ";
                }
                System.out.print(s);
            }
            System.out.println();
        }
    }
    
    private static int[][] Q = new int[6][6];
    static
    {
        Q[0][0] = 0;
        Q[0][1] = 0;
        Q[0][2] = 0;
        Q[0][3] = 0;
        Q[0][4] = 0;
        Q[0][5] = 0;
        
        Q[1][0] = 0;
        Q[1][1] = 0;
        Q[1][2] = 0;
        Q[1][3] = 0;
        Q[1][4] = 0;
        Q[1][5] = 0;
        
        Q[2][0] = 0;
        Q[2][1] = 0;
        Q[2][2] = 0;
        Q[2][3] = 0;
        Q[2][4] = 0;
        Q[2][5] = 0;
        
        Q[3][0] = 0;
        Q[3][1] = 0;
        Q[3][2] = 0;
        Q[3][3] = 0;
        Q[3][4] = 0;
        Q[3][5] = 0;
        
        Q[4][0] = 0;
        Q[4][1] = 0;
        Q[4][2] = 0;
        Q[4][3] = 0;
        Q[4][4] = 0;
        Q[4][5] = 0;
        
        Q[5][0] = 0;
        Q[5][1] = 0;
        Q[5][2] = 0;
        Q[5][3] = 0;
        Q[5][4] = 0;
        Q[5][5] = 0;
    }
}

经过500次计算得到如下结果

第499次学习, 初始房间是1
0    0    0    0    396  0    
0    0    0    316  0    496  
0    0    0    316  0    0    
0    396  252  0    396  0    
316  0    0    316  0    496  
0    396  0    0    396  496  

此时，我们从任意一个房间进入，每次选取最高分值步骤移动，总可以找到最短的逃离路径。