具有优先体验重放功能和目标网络的DQN不会得到改善

Question

I tried to code a neural network to solve OpenAI's CartPole environment with Tensorflow and Keras. The network uses prioritized experience replay and a seperate target network which is updated every tenth episode. Here's the code:

import numpy as np
import gym
import matplotlib.pyplot as plt
from collections import deque

from tensorflow import keras

class agent:
    def __init__(self,inputs,outputs,gamma):
        self.gamma = gamma
        self.epsilon = 1.0
        self.epsilon_decay = 0.999
        self.epsilon_min = 0.01
        self.inputs = inputs
        self.outputs = outputs
        self.network = self._build_net()
        self.target_network = self._build_net()
        self.replay_memory = deque(maxlen=100000)

        self.target_network.set_weights(self.network.get_weights())

    def _build_net(self):
        model = keras.models.Sequential()

        model.add(keras.layers.Dense(10,activation='relu',input_dim=self.inputs))
        model.add(keras.layers.Dense(10,activation='relu'))
        model.add(keras.layers.Dense(10,activation='relu'))
        model.add(keras.layers.Dense(self.outputs,activation='sigmoid'))
        model.compile(optimizer='adam',loss='categorical_crossentropy')

        return model

    def act(self,state,testing=False):
        if not testing:
            if self.epsilon > np.random.rand():
                return np.random.randint(self.outputs)
            else:
                return np.argmax(self.network.predict(np.array([state])))

        else:
            return np.argmax(self.network.predict(np.array([state])))

    def remember(self,state,action,next_state,reward,win):
        self.replay_memory.append((state,action,next_state,reward,win))

    def get_batch(self):
        batch = []
        losses = []
        targets = []

        if len(self.replay_memory) >= 32:
            for state,action,next_state,reward,done in self.replay_memory:
                target_f = np.zeros([1,self.outputs])
                if done != False:
                    target = (reward + self.gamma * np.amax(self.target_network.predict(np.array([next_state]))[0]))
                    target_f[0][action] = target

                targets.append(target_f)
                loss = np.mean(self.network.predict(np.array([state]))-target_f)**2
                losses.append(loss)

            indexes = np.argsort(losses)[:32]

            for indx in indexes:
                batch.append((self.replay_memory[indx][0], targets[indx]))

            return batch

    def replay(self,batch):
        for state,target in batch:
            self.network.fit(np.array([state]), target, epochs=1, verbose=0)
        self.epsilon = max(self.epsilon_min, self.epsilon*self.epsilon_decay)

    def update_target_network(self):
        self.target_network.set_weights(self.network.get_weights())

    def save(self):
        self.network.save_weights('./DQN.model')

    def load(self):
        self.network.load_weights('./DQN.model')

env = gym.make('CartPole-v0')
episodes = 1000
agent = agent(env.observation_space.shape[0],env.action_space.n,0.99)
rewards = []

state = env.reset()
for _ in range(500):
    action = agent.act(state,1.0)
    next_state, reward, win, _ = env.step(action)
    agent.remember(state,action,next_state,reward,win)
    state = next_state

    if win:
        state = env.reset()

for e in range(episodes):
    print('episode:',e+1)
    batch = agent.get_batch()
    state = env.reset()
    for t in range(400):
        action = agent.act(state)
        next_state, reward, win, _ = env.step(action)
        agent.remember(state,action,next_state,reward,win)
        state = next_state

        if win:
            break

        agent.replay(batch)

    print('score:',t)
    print('epsilon:',agent.epsilon)
    print('')

    if e%10 == 0:
        agent.update_target_network()
        agent.save()

    rewards.append(t)
    plt.plot(list(range(e+1)),rewards)
    plt.savefig('./reward.png')

The problem is that the agent gets worse as epsilon decreases. And when epsilon is a its lowest value, the agent only gets through 7-9 steps before the Pole falls, as seen in the Image below. Can someone tell me why my agent isn't learning anything and how to fix it?

Answer 1

Your network output should be using linear as activation as opposed to sigmoid , since you are bootstrapping summation of all future rewards. The loss should be changed to mse as well, according to the derivation for the td error from the original paper.

Also, if the experience ended in a terminal state (done==True), the td target for the action taken should set equal to the reward (you are currently setting it as 0).

Answer 2

Try making the reward = -reward if done. It helps