DQN：OpenAI Gym LunarLander 环境 DeepQ 学习失败

Question

我正在尝试为 LunarLander 环境实施 DQN 代理。 尽管一切看起来都完好无损，但 model 始终表现不佳。 甚至看起来代理人正试图做得越来越糟。 有时它看起来很随意。

剧情奖励下降

更奇怪的是 model deos 看起来不像是随机动作，一些动作使用稳步增加。 （检查下图：触发底部引擎动作不断被使用，而无所事事动作使用减少）

动作用法

我尝试了许多不同的HP。 我想，我在这里忽略了一些非常基本的东西，或者我的代码中有一个错误。 这是我的代码的要点。 如果有人可以提供帮助，我会很高兴。

我试过的HP：

 - lr = 0.01, 0.001, 0.0001, 0.00001 
 - bs = 32, 64, 128, 256
 - gamma = 0.99, 0.999
 - eps_start = 1.0
 - eps_end = 0.05
 - eps_decay = 0.99, 0.999
 - target_update = 5
 - tau = 0.01, 0.001, 0.0001
 - buffer_size = 10000, 100000
 - max_timestep = 1000, 10000

代理人：

class Agent():

def __init__(self, state_size, action_size, args):

    self.state_size = state_size
    self.action_size = action_size

    self.args = args


    #Q-Network
    self.Q = QNetwork(state_size, action_size, self.args.device).to(self.args.device)
    self.Q_target = QNetwork(state_size, action_size, self.args.device).to(self.args.device)
    self.optimizer = optimizer.Adam(self.Q.parameters(), self.args.lr)

    self.memory = ReplayBuffer(action_size, args)

    self.time_step = 0

def step(self, state, action, reward, next_state, done):
    self.memory.add(state, action, reward, next_state, done)

    self.time_step = (self.time_step + 1) % self.args.target_update

    if self.time_step == 0:
        if len(self.memory) > self.args.batch_size:
            experiences = self.memory.sample()
            self.learn(experiences)

def act(self, state, eps):
    state = torch.from_numpy(state).float().unsqueeze(0).to(self.args.device)
    self.Q.eval()
    with torch.no_grad():
        action_values = self.Q(state)
    self.Q.train()

    if random() > eps:
        return np.argmax(action_values.cpu().data.numpy())
    else:
        return choice(np.arange(self.action_size), p=[0.25, 0.25, 0.25, 0.25])

def learn(self, experiences):

    states, actions, rewards, next_states, dones = experiences

    ### Extract next maximum estimated value from target network
    Q_targets_next = self.Q_target(next_states).detach().max(1)[0].unsqueeze(1)
    ### Calculate target value from bellman equation
    Q_targets = rewards.cpu() + (self.args.gamma * Q_targets_next.cpu() * (1-dones.cpu()))
    ### Calculate expected value from local network
    Q_expected = self.Q(states).cpu().gather(1, actions.cpu())

    Q_expected.to(self.args.device)
    Q_targets.to(self.args.device)
    loss = F.mse_loss(Q_expected, Q_targets)
    self.optimizer.zero_grad()
    loss.backward()
    self.optimizer.step()

    self.soft_update(self.Q, self.Q_target)

def soft_update(self, Q, Q_target):

    for Q_target_params, Q_params in zip(Q_target.parameters(), Q.parameters()):
        Q_target_params.data.copy_(self.args.tau * Q_params.data + (1.0-self.args.tau) * Q_target_params.data)

训练循环：

def run_episodes_train(env, agent, stats, tensorboard, args):
top_score = float("-inf")
scores_window = deque(maxlen=100)
eps = args.eps_start

for i_episode in range(1, args.n_episodes+1):
    state = env.reset()
    score = 0
    for t in range(args.max_timestep):
        action = agent.act(state, eps)
        next_state, reward, done, _, _ = env.step(action)
        stats.step(reward, action)

        agent.step(state, action, reward, next_state, done)
        state = next_state
        score += reward
        if done:
            break
    tensorboard.write_episode_data(i_episode, eval_dict={"episode_reward": score,
                                                 "do nothing": stats.get_action_usage(0),
                                                 "left": stats.get_action_usage(1),
                                                 "right": stats.get_action_usage(3),
                                                 "bottom": stats.get_action_usage(2),
                                                 "eps": eps})

    eps = max(args.eps_end, args.eps_decay*eps)
    print("\nEpisode: ", i_episode)

    if i_episode % 10 == 0 and score > top_score:
        top_score = score
        agent.Q.to("cpu")
        torch.save(agent.Q.state_dict(), args.model_dir+"Q_checkpoint.pt")
        print("\nAgent saved...")
        agent.Q.to(args.device)
return