[RL學習篇][#3] 自動學習grid_mdp最佳的策略

本文修改 policy_iteration.py程式，讓他可以執行[#1]的程式，並找出最佳動作。

# /bin/python
import numpy;
import random;
import gym;
#from grid_mdp import Grid_Mdp


class Policy_Value:
    def __init__(self, grid_mdp):
        self.v = [0.0 for i in range(len(grid_mdp.env.states) + 1)] # 初始變數v <-- 值函數

        self.pi = dict()
        for state in grid_mdp.env.states:
            if state in grid_mdp.env.terminate_states: continue
            self.pi[state] = grid_mdp.env.action_s[0] #初始pi <-- 策略pi

    def policy_improve(self, grid_mdp):

        for state in grid_mdp.env.states:
            grid_mdp.env.setAction(state)  # upate state
            if state in grid_mdp.env.terminate_states: continue

            a1 = grid_mdp.env.action_s[0]
            s, r, t, z = grid_mdp.env._step(a1)
            v1 = r + grid_mdp.env.gamma * self.v[s]

            for action in grid_mdp.env.action_s:
                s, r, t, z = grid_mdp.env._step(action)
                if v1 < r + grid_mdp.env.gamma * self.v[s]: # 當action有更好的值，則更新動作
                    a1 = action
                    v1 = r + grid_mdp.env.gamma * self.v[s]

            self.pi[state] = a1   # 紀錄最佳動作

    def policy_evaluate(self, grid_mdp):
        for i in range(1000):
            delta = 0.0
            for state in grid_mdp.env.states:
                grid_mdp.env.setAction(state) # upate state
                if state in grid_mdp.env.terminate_states: continue
                action = self.pi[state]

                s, r, t, z = grid_mdp.env.step(action)
                new_v = r + grid_mdp.env.gamma * self.v[s]
                delta += abs(self.v[state] - new_v)
                self.v[state] = new_v

            if delta < 1e-6:
                break;

    def policy_iterate(self, grid_mdp):
        for i in range(100):
            self.policy_evaluate(grid_mdp);
            self.policy_improve(grid_mdp);


if __name__ == "__main__":
    #grid_mdp = Grid_Mdp()
    env = gym.make('GridWorld-v0')

    policy_value = Policy_Value(env)
    policy_value.policy_iterate(env)
    print("value:")
    for i in range(1, 6):
        print("%d:%f	" % (i, policy_value.v[i]), )
    print("")

    print("policy:")
    for i in range(1, 6):
        print("%d->%s	" % (i, policy_value.pi[i]), )
    print("")

執行結果如下:

-----------------------------------------------------------------------------------------------------------------------------------------------------

/home/lsa-dla/anaconda3/envs/tensorflow/bin/python /home/lsa-dla/PycharmProjects/grid_mdp/lsa_test2.py
WARN: Environment '<class 'gym.envs.classic_control.grid_mdp.GridEnv'>' has deprecated methods. Compatibility code invoked.
value:
1:0.640000
2:0.800000
3:1.000000
4:0.800000
5:0.640000

policy:
1->e
2->e
3->s
4->w
5->w

Process finished with exit code 0

 ------------------------------------------------------------------------------------------------------------------------------------------------------

reference:

[1]  Reinforcement_Learning_Blog/2.强化学习系列之二:模型相关的强化学习/