TypeError：apply_gradients() 得到了一个意外的关键字参数“global_step”

Question

After days trying to make one RL agent, I finally succeeded in creating its experience, but when I try to train it I get this error. I've tried all I could: different experience, changed step params... I am just out of ideas.

import pyxinput
import time
import cv2
from PIL import ImageGrab
import numpy as np
import keyboard
import tensorflow
import tf_agents
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import torch
#from tf_agents.networks import actor_distribution_networ

from tf_agents.policies import random_py_policy

Tensod_spec = tf_agents.specs.BoundedArraySpec(
   (15,),
   dtype=np.float32,
   name="XimputSpecs",
   minimum=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
   maximum=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
)

Tensod_spec2 = tf_agents.specs.TensorSpec(
   [440, 600, 1], dtype=tf.int32, name="ScreenSpecs"
)

Tensor_reward_spe = tf_agents.specs.TensorSpec(
   [1, 1], dtype=tf.int32, name="Reward"
)

FromEnv = tf_agents.specs.BoundedTensorSpec(
   shape=(440, 600, 1),
   dtype='uint8',
   name='observation',
   minimum=0,
   maximum=255
)
FromEnv2 = tf_agents.specs.BoundedTensorSpec(
   shape=(1, 440, 600, 1),
   dtype=tf.int32,
   name='observation',
   minimum=0,
   maximum=255
)

fullscreen = [110, 130, 710, 570]

screenpil = ImageGrab.grab(bbox=fullscreen)
showprint = np.array(screenpil)
grayscreen = cv2.cvtColor(showprint, cv2.COLOR_BGR2GRAY)
screenrect = cv2.cvtColor(grayscreen, cv2.COLOR_GRAY2BGR)
grayscreen = grayscreen.reshape(440, 600, 1)

time_step_spec2 = tf_agents.trajectories.time_step.time_step_spec(
   observation_spec=FromEnv,
   #reward_spec = Tensor_reward_spec
)

time_step_spec = tf_agents.trajectories.time_step.time_step_spec(
    observation_spec=FromEnv,
    #reward_spec = Tensor_reward_spec
)

actor_net = tf_agents.networks.actor_distribution_network.ActorDistributionNetwork(
   input_tensor_spec=FromEnv,
   output_tensor_spec=tf_agents.specs.tensor_spec.from_spec(Tensod_spec),
   activation_fn='relu',
   #conv_layer_params=[(25, 40, 2)],
   fc_layer_params=(50, 25, 15),
   #dtype='int32'
)
print(actor_net)

train_step_counter = tf.dtypes.cast(1, tf.int32)

optimizer = tf.keras.optimizers.Adam(learning_rate=0.003)

tf_agent = tf_agents.agents.ReinforceAgent(
   time_step_spec=time_step_spec,
   action_spec=tf_agents.specs.tensor_spec.from_spec(Tensod_spec),
   actor_network=actor_net,
   optimizer=optimizer,
   normalize_returns=True,
   #train_step_counter=tf.Variable(1, name="global_step")
   )
tf_agent.initialize()

grayscreen2 = grayscreen
grayscreen2 = grayscreen2.reshape(1, 440, 600, 1)
time_step2 = tf_agents.trajectories.time_step.TimeStep(
   step_type=tf_agents.trajectories.time_step.StepType.FIRST,
   reward=tf.dtypes.cast(1, tf.float32),
   discount=tf.dtypes.cast(1, tf.float32),
   observation=grayscreen2
)

policy_state = tf_agent.policy.get_initial_state(batch_size=1)

policy_step = tf_agent.policy.action(time_step2, policy_state)
print(policy_step)

observe = time_step2.observation
#print(observe.dtype)
#observe = observe.astype(int)
#print(observe.shape)

experience = tf_agents.trajectories.trajectory.Trajectory(
   action=tf.compat.v2.Variable([
           tf.compat.v2.Variable(policy_step.action),
           tf.compat.v2.Variable(policy_step.action),
           tf.compat.v2.Variable(policy_step.action)
   ]),
   reward=tf.compat.v2.Variable([[
       tf.compat.v2.Variable(time_step2.reward),
       tf.compat.v2.Variable(time_step2.reward),
       tf.compat.v2.Variable(time_step2.reward)
   ]]),
   step_type=tf.compat.v2.Variable([[
       tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.FIRST),
       tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.MID),
       tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.LAST)
   ]]),
   observation=tf.compat.v2.Variable([
       tf.compat.v2.Variable(observe),
       tf.compat.v2.Variable(observe),
       tf.compat.v2.Variable(observe)
   ]),
   policy_info=tf_agent.policy.info_spec,
   next_step_type=tf.compat.v2.Variable([[
       tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.MID),
       tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.LAST),
       tf.compat.v2.Variable(tf_agents.trajectories.time_step.StepType.LAST)
   ]]),
   discount=tf.compat.v2.Variable([[
       tf.dtypes.cast(1, tf.float32),
       tf.dtypes.cast(1, tf.float32),
       tf.dtypes.cast(1, tf.float32)
   ]]), 
)

train_loss = tf_agent.train(experience)
print(train_loss)

And I get this error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-15-4dd3966a32b6> in <module>
      1 #
----> 2 train_loss = tf_agent.train(experience)
      3 print(train_loss)

~\AppData\Local\Programs\Python\Python38\lib\site-packages\tf_agents\agents\tf_agent.py in train(self, experience, weights, **kwargs)
    516 
    517     if self._enable_functions:
--> 518       loss_info = self._train_fn(
    519           experience=experience, weights=weights, **kwargs)
    520     else:

~\AppData\Local\Programs\Python\Python38\lib\site-packages\tf_agents\utils\common.py in with_check_resource_vars(*fn_args, **fn_kwargs)
    183         # We're either in eager mode or in tf.function mode (no in-between); so
    184         # autodep-like behavior is already expected of fn.
--> 185         return fn(*fn_args, **fn_kwargs)
    186       if not resource_variables_enabled():
    187         raise RuntimeError(MISSING_RESOURCE_VARIABLES_ERROR)

~\AppData\Local\Programs\Python\Python38\lib\site-packages\tf_agents\agents\reinforce\reinforce_agent.py in _train(self, experience, weights)
    286                                           self.train_step_counter)
    287 
--> 288     self._optimizer.apply_gradients(
    289         grads_and_vars, global_step=0)
    290 

TypeError: apply_gradients() got an unexpected keyword argument 'global_step'

What is this global step, and where is this error coming from? Why can't I train my agent?

Specs:

• Python 3.8
• TensorFlow 2.4 (GPU and non-GPU)
• Windows 10 / ubuntu

If you need more info, please let me know.

EDIT: Tried other agents they run fine and i posted this ISUE on Tensor GIT: https://github.com/tensorflow/tensorflow/issues/48424 If anyone has the same problem in the future

Answer 1

You should try to use a different Optimizer. Those in tf.keras.optimizer don't take global_steps as an argument in apply_gradients function. Instead, use these from tf.compat.v1.train , eg,

optimizer = tf.compat.v1.train.AdamOptimizer(learn_rate=0.003)

Note this passes the runtime check, but it makes the training impossible to complete. global_step is supposed to take a Variable and its value will be +1 when apply_gradients is called. However, here you see global_step=0 is passed in making it no effect at all. The train_step_counter you defined above will remain 0 .

Also note there's a fix on the way.