using list in creating pytorch NN module

Question

This code runs fine to create a simple feed-forward neural Network. The layer ( torch.nn.Linear ) is assigned to the class variable by using self .

class MultipleRegression3L(torch.nn.Module):
  def __init__(self, num_features):
    super(MultipleRegression3L, self).__init__()

    self.layer_1 = torch.nn.Linear(num_features, 16)
    ## more layers

    self.relu = torch.nn.ReLU()

  def forward(self, inputs):
    x = self.relu(self.layer_1(inputs))
    x = self.relu(self.layer_2(x))
    x = self.relu(self.layer_3(x))
    x = self.layer_out(x)
    return (x)

  def predict(self, test_inputs):
    return self.forward(test_inputs)

However, when I tried to store the layer using the list:

class MultipleRegression(torch.nn.Module):
  def __init__(self, num_features, params):
    super(MultipleRegression, self).__init__()

    number_of_layers = 3 if not 'number_of_layers' in params else params['number_of_layers']
    number_of_neurons_in_each_layer = [16, 32, 16] if not 'number_of_neurons_in_each_layer' in params else params['number_of_neurons_in_each_layer']
    activation_function = "relu" if not 'activation_function' in params else params['activation_function']

    self.layers = []
    v1 = num_features
    for i in range(0, number_of_layers):
        v2 = number_of_neurons_in_each_layer[i]
        self.layers.append(torch.nn.Linear(v1, v2))
        v1 = v2

    self.layer_out = torch.nn.Linear(v2, 1)

    if activation_function == "relu":
        self.act_func = torch.nn.ReLU()
    else:
        raise Exception("Activation function %s is not supported" % (activation_function))

  def forward(self, inputs):
    x = self.act_func(self.layers[0](inputs))

    for i in range(1, len(self.layers)):
        x = self.act_func(self.layers[i](x))

    x = self.layer_out(x)

    return (x)

The two models do not behave the same way. What can be wrong here?

Answer 1

Pytorch needs to keep the graph of the modules in the model, so using a list does not work. Using self.layers = torch.nn.ModuleList() fixed the problem.