ValueError：檢查輸入時出錯：預期dense_input有2維，但得到形狀為(1,1,2)的數組

Question

編輯：問題已解決。 解決方法如下。

嘗試構建 RL 模型來處理任務。 有兩個輸入：x 和 y，都是在 1 到 100 的整數范圍內測量的。基於這兩個輸入，應該有一個輸出（要采取的行動，離散（5））和置信度。

另外，我對這個領域很陌生。 請隨時問我任何事情或糾正我一些看起來完全愚蠢/錯誤的事情。

這是我的程序（導入尚未清理....）：

from abc import ABC
import gym
from tensorflow import keras
from gym import Env
from gym.spaces import Discrete, Box
import random
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers, losses, metrics
from tensorflow.keras.layers import Dense, Flatten, Input
from tensorflow.keras.optimizers import Adam
import os
from rl.agents import DQNAgent
from rl.policy import BoltzmannQPolicy
from rl.memory import SequentialMemory
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

steps = 10000
episodes = 100
score_requirement = 1000

class PlantEnv(Env, ABC):
    def __init__(self):
        # Actions = water: 0=(none), 1=(3 seconds), 2=(4 seconds), 3=(5 seconds), 4=(6 seconds)
        self.action_space = Discrete(5)

        # Starting Moisture
        moisture = 20 + random.randint(-10, 10)
        # Starting Chance of Rain
        chance_of_rain = 50 + random.randint(-50, 50)

        # Observations
        self.observation_space = Box(low=np.array([0, 0]), high=np.array([100, 100]), dtype=np.int)
        self.state = moisture, chance_of_rain

        # Number of water steps left
        self.water_length = steps

    def step(self, action):
        # Action section
        water = 0

        if action == 1:
            water = 2
        elif action == 2:
            water = 3
        elif action == 3:
            water = 4
        elif action == 4:
            water = 5

        moisture, chance_of_rain = self.state

        moisture += (water * 5)
        self.water_length -= 1

        # Reward Section
        reward = 0
        if 40 <= moisture <= 60:
            reward = 2
        # If moisture is dry or wet
        elif 60 < moisture <= 80 or 20 <= moisture < 40:
            reward = 0.5
        # If moisture is really dry or really wet
        elif 80 < moisture <= 100 or 0 <= moisture < 20:
            reward = -1
        # If moisture is really dry or really wet
        elif 100 < moisture or moisture < 0:
            reward = -2

        # Check if shower is done
        if self.water_length <= 0:
            done = True
        else:
            done = False

        moistureLoss = random.randint(15, 25)
        moisture -= moistureLoss
        chance_of_rain = 50 + random.randint(-50, 50)
        xfactor = chance_of_rain + random.randint(-50, 50)
        if xfactor > 100:
            moisture += (10 + random.randint(0, 15))

        # Set placeholder for info
        info = {}

        # Save current state
        self.state = moisture, chance_of_rain

        # Return step information
        return self.state, reward, done, info

    def reset(self):
        # Reset test environment
        # Set starting moisture
        moisture = 50 + random.randint(-10, 10)
        # Set starting chance of rain array
        chance_of_rain = 50 + random.randint(-50, 50)
        self.state = moisture, chance_of_rain
        # Reset Test time
        self.water_length = steps
        return self.state


def build_model():
    model = Sequential()
    model.add(Flatten(input_shape=(1, 4)))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(2, activation='linear'))
    return model


def build_agent(model):
    policy = BoltzmannQPolicy()
    memory = SequentialMemory(limit=50000, window_length=1)
    dqn = DQNAgent(model=model, memory=memory, policy=policy, nb_actions=2,
                   nb_steps_warmup=10, target_model_update=1e-2)
    return dqn


# Create environment
env = PlantEnv()

accepted_scores = []
training_data = []
scores = []
good_episodes = 0

# Create episodes and initiate simulation
for episode in range(1, episodes + 1):
    observation = env.reset()
    done = False
    score = 0
    history = []
    prev_observation = []

    while not done:
        action = env.action_space.sample()
        if observation[0] > 100:
            action = 0
        elif observation[0] < 0:
            action = 4
        observation, reward, done, info = env.step(action)
        score += reward
        if len(prev_observation) > 0:
            history.append([prev_observation, action])
        prev_observation = observation

    if score >= score_requirement:
        good_episodes += 1
        accepted_scores.append(score)
        for data in history:
            if data[1] == 1:
                output = [1]
            else:
                output = [0]

            training_data.append([data[0], output])

    scores.append(score)

if len(accepted_scores) > 0:
    print("Average accepted score: ", np.mean(accepted_scores))
    print("Median accepted score : ", np.median(accepted_scores))
print("Episodes above accepted score of {}: {}/{}\n".format(score_requirement, good_episodes, episodes))

model = build_model()
model.summary()

dqn = build_agent(model)
dqn.compile(Adam(learning_rate=1e-3), metrics=['mae'])
dqn.fit(env, nb_steps=50000, visualize=False, verbose=1)

第一個模型在嘗試 dqn.fit 時出現此錯誤： ValueError：檢查輸入時出錯：預期密集輸入有 2 維，但得到形狀為 (1, 1, 2) 的數組

第二個模型在嘗試 build_agent 時出現此錯誤： AttributeError: 'list' object has no attribute 'shape'

關於我做錯了什么或如何糾正它的任何想法都將是一個巨大的幫助。 我有 95% 的把握確信我的環境設置正確。

我最初使用第一個模型只是為了看看我是否可以編譯和運行程序。 然后，經過進一步的研究，我建立了第二個模型，因為我知道它能夠給我一個有信心評級的行動。 在兩個轉彎處都出錯。

Answer 1

您的模型需要 2D 輸入，但您將其定義為 1D。 這是一個工作示例：

from abc import ABC
import gym
from tensorflow import keras
from gym import Env
from gym.spaces import Discrete, Box
import random
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers, losses, metrics
from tensorflow.keras.layers import Dense, Flatten, Input
from tensorflow.keras.optimizers import Adam
import os
from rl.agents import DQNAgent
from rl.policy import BoltzmannQPolicy
from rl.memory import SequentialMemory
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

steps = 10000
episodes = 100
score_requirement = 1000

class PlantEnv(Env, ABC):
    def __init__(self):
        # Actions = water: 0=(none), 1=(3 seconds), 2=(4 seconds), 3=(5 seconds), 4=(6 seconds)
        self.action_space = Discrete(5)

        # Starting Moisture
        moisture = 20 + random.randint(-10, 10)
        # Starting Chance of Rain
        chance_of_rain = 50 + random.randint(-50, 50)

        # Observations
        self.observation_space = Box(low=np.array([0, 0]), high=np.array([100, 100]), dtype=np.int)
        self.state = moisture, chance_of_rain

        # Number of water steps left
        self.water_length = steps

    def step(self, action):
        # Action section
        water = 0

        if action == 1:
            water = 2
        elif action == 2:
            water = 3
        elif action == 3:
            water = 4
        elif action == 4:
            water = 5

        moisture, chance_of_rain = self.state

        moisture += (water * 5)
        self.water_length -= 1

        # Reward Section
        reward = 0
        if 40 <= moisture <= 60:
            reward = 2
        # If moisture is dry or wet
        elif 60 < moisture <= 80 or 20 <= moisture < 40:
            reward = 0.5
        # If moisture is really dry or really wet
        elif 80 < moisture <= 100 or 0 <= moisture < 20:
            reward = -1
        # If moisture is really dry or really wet
        elif 100 < moisture or moisture < 0:
            reward = -2

        # Check if shower is done
        if self.water_length <= 0:
            done = True
        else:
            done = False

        moistureLoss = random.randint(15, 25)
        moisture -= moistureLoss
        chance_of_rain = 50 + random.randint(-50, 50)
        xfactor = chance_of_rain + random.randint(-50, 50)
        if xfactor > 100:
            moisture += (10 + random.randint(0, 15))

        # Set placeholder for info
        info = {}

        # Save current state
        self.state = moisture, chance_of_rain

        # Return step information
        return self.state, reward, done, info

    def reset(self):
        # Reset test environment
        # Set starting moisture
        moisture = 50 + random.randint(-10, 10)
        # Set starting chance of rain array
        chance_of_rain = 50 + random.randint(-50, 50)
        self.state = moisture, chance_of_rain
        # Reset Test time
        self.water_length = steps
        return self.state


def build_model():
    model = Sequential()
    model.add(Flatten(input_shape=(1, 2)))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(24, activation='relu'))
    model.add(Dense(5, activation='linear'))
    return model


def build_agent(model):
    policy = BoltzmannQPolicy()
    memory = SequentialMemory(limit=50000, window_length=1)
    dqn = DQNAgent(model=model, memory=memory, policy=policy, nb_actions=5,
                   nb_steps_warmup=10, target_model_update=1e-2)
    return dqn


# Create environment
env = PlantEnv()

accepted_scores = []
training_data = []
scores = []
good_episodes = 0

# Create episodes and initiate simulation
for episode in range(1, episodes + 1):
    observation = env.reset()
    done = False
    score = 0
    history = []
    prev_observation = []

    while not done:
        action = env.action_space.sample()
        if observation[0] > 100:
            action = 0
        elif observation[0] < 0:
            action = 4
        observation, reward, done, info = env.step(action)
        score += reward
        if len(prev_observation) > 0:
            history.append([prev_observation, action])
        prev_observation = observation

    if score >= score_requirement:
        good_episodes += 1
        accepted_scores.append(score)
        for data in history:
            if data[1] == 1:
                output = [1]
            else:
                output = [0]

            training_data.append([data[0], output])

    scores.append(score)

if len(accepted_scores) > 0:
    print("Average accepted score: ", np.mean(accepted_scores))
    print("Median accepted score : ", np.median(accepted_scores))
print("Episodes above accepted score of {}: {}/{}\n".format(score_requirement, good_episodes, episodes))

model = build_model()
model.summary()

dqn = build_agent(model)
dqn.compile(Adam(learning_rate=1e-3), metrics=['mae'])
dqn.fit(env, nb_steps=50000, visualize=False, verbose=1)

請注意， input_shape 4 等於states數，輸出節點 (2) 等於actions 。 您必須根據您的數據集更改這些參數。