如何在 PyTorch 中实现 ResNet50？

Question

I learn NN in Coursera course, by deeplearning.ai and for one of my homework was an assignment for ResNet50 implementation by using Keras, but I see Keras is too high-level language) and decided to implement it in the more sophisticated library - PyTorch. I recorded it, but something went wrong. May someone, please, say to me what's going on and why there is appeared error with parameters when I cause method ResNet.parameters() when putting it in Adam optimization.

class implementation:

class ResNet50(torch.nn.Module):
    def __init__(self, input_shape = (3, 96, 96), classes = 10):
        super(ResNet50, self).__init__()    
        """
        Implementation of the popular ResNet50 the following architecture:
        Conv2d -> BatchNorm -> ReLU -> MaxPool -> ConvBlock - > IdBlock*2 - > convBlock -> IdBlock*3 -> ConvBlock -> IdBlock*5 -> ConvBlock -> IdBlock*2 -> AvgPool -> FCLayer
    
        Arguments:
        input_shape -- shape of the image of the dataset
        classes -- integer, number of classes    
        """
    
        self.input_shape = input_shape
        self.classes = classes
        self.relu = torch.nn.ReLU() 
        
    def identity_block(self, X, f, filters):
        # Notice that there is no any kind of Pooling.
        """
        Implementation of the identity block.
    
        Arguments:
        X -- input tensor of shape(m , n_H_prev, n_W_prev, n_C_prev)
        f -- integer, specifying the shape of the middle CONV's window for the main path
        filters -- python list of integers, defining the number of filters in the CONV layers of the main path
        
        Returns:
        X -- output of the identity block, tensor of shape (n_H, n_W, n_C)
        """
    
        # Retrieve Filters
        F1, F2, F3 = filters
    
        # Save the input value. It will be needed later to be added back to the main path.
        X_shortcut = X
    
        # First component of the main path
        X = torch.nn.Conv2d(in_channels=X.shape[0], out_channels=F1, kernel_size=1, stride=1, padding=0)(X)
        X = torch.nn.BatchNorm2d(num_features=F1)(X)
        X = self.relu(X)
    
        # Second component of the main path
        X = torch.nn.Conv2d(in_channels=F1, out_channels=F2, kernel_size=f, stride=1, padding=f//2)(X)
        X = torch.nn.BatchNorm2d(num_features=F2)(X)
        X = self.relu(X)
    
        # Third component of the main path
        X = torch.nn.Conv2d(in_channels=F2, out_channels=F3, kernel_size=1, stride=1, padding=0)(X)
        X = torch.nn.BatchNorm2d(num_features=F3)(X)
        # X = self.relu(X) - NO RELU, notice this!
    
        # Final step: Add shortcut value to main path, and pass it through a ReLU
        X = X_shortcut + X
        X = self.relu(X)

        return X 
    def convolution_block(self, X, f, filters, s = 2):
        # Notice that here is no any kind of Pooling.
        """
        Implementation of the convolutional block.
    
        Arguments:
        X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
        f -- integer, specifying the shape of middle CONV's window for main path
        filters -- python list of integers, defining the number of filters in the CONV layers of the main path
        s -- integer, specifying the stride to be used
    
        Returns:
        X -- output of the convolution block, tensor of shape (n_H, n_W, n_C)
        """
    
        # Retrieve Filters
        F1, F2, F3 = filters
    
        # Save the input value
        X_shortcut = X
    
        # First component of the main path
        X = torch.nn.Conv2d(in_channels=X.shape[0], out_channels=F1, kernel_size=1, stride=s, padding=0)(X)
        X = torch.nn.BatchNorm2d(num_features=F1)(X)
        X = self.relu(X)
    
        # Second component of the main path
        X = torch.nn.Conv2d(in_channels=F1, out_channels=F2, kernel_size=f, stride=1, padding=f//2)(X)
        X = torch.nn.BatchNorm2d(num_features=F2)(X)
        X = self.relu(X)
    
        # Third component of the main path
        X = torch.nn.Conv2d(in_channels=F2, out_channels=F3, kernel_size=1, stride=1, padding=0)(X)
        X = torch.nn.BatchNorm2d(num_features=F2)(X)
        # X = self.relu(X) - NO RELU, notice this!
    
        # Shortcut path
        X_shortcut = torch.nn.Conv2d(in_channels=X_shortcut.shape[0], out_channels=F3, kernel_size=1, stride=s, padding=0)(X)
        X_shortcut = torch.nn.BatchNorm2d(num_features=F3)(X)
        # X = self.relu(X) - NO RELU, notice this!
    
        # Final step: Add shortcut value to main path, and pass it through a ReLU
        X = X_shortcut + X
        X = self.relu(X)
    
        return X
    def forward(self, X):
        """
        Forward propogation by the following architecture:
        Conv2d -> BatchNorm -> ReLU -> MaxPool -> ConvBlock - > IdBlock*2 - > convBlock -> IdBlock*3 -> ConvBlock -> IdBlock*5 -> ConvBlock -> IdBlock*2 -> AvgPool -> FCLayer
    
        Arguments:
        X -- input data for Network that needed to be propagated
    
        Returns:
        X -- output of the ResNet50, that propagated through it
        """
    
        # # Define the input as a tensor with shape self.input_shape
        # X = torch.zeros_like(self.input_shape)
    
        # Stage 1 
        X = torch.nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3)(X) # 96x96x3 -> 48x48x64
        X = torch.nn.BatchNorm2d(num_features=64)(X)
        X = self.relu(X)
        X = torch.nn.MaxPool2d(kernel_size=3, stride=2, padding=0)(X) # 48x48x64 -> 23x23x64
    
        # Stage 2
        X = self.convolution_block(X, f=3, filters=[64, 64, 256], s=1) # 23x23x64 -> 23x23x256
        X = self.identity_block(X, 3, [64, 64, 256]) # same
        X = self.identity_block(X, 3, [64, 64, 256]) # same
    
        # Stage 3
        X = self.convolution_block(X, f=3, filters=[128, 128, 512], s=2) # 23x23x256 -> 12x12x512
        X = self.identity_block(X, 3, [128, 128, 512]) # same
        X = self.identity_block(X, 3, [128, 128, 512]) # same
        X = self.identity_block(X, 3, [128, 128, 512]) # same
    
        # Stage 4
        X = self.convolution_block(X, f=3, filters=[256, 256, 1024], s=2) # 12x12x512 -> 6x6x1024
        X = self.identity_block(X, 3, [256, 256, 1024]) # same
        X = self.identity_block(X, 3, [256, 256, 1024]) # same
        X = self.identity_block(X, 3, [256, 256, 1024]) # same
        X = self.identity_block(X, 3, [256, 256, 1024]) # same
        X = self.identity_block(X, 3, [256, 256, 1024]) # same
    
        # Stage 5
        X = self.convolution_block(X, f=3, filters=[512, 512, 2048], s=2) # 6x6x1024 -> 3x3x2048
        X = self.identity_block(X, 3, [512, 512, 2048]) # same
        X = self.identity_block(X, 3, [512, 512, 2048]) # same
    
        # AvgPool
        X = torch.nn.AvgPool2d(kernel_size=2)(X) # 3x3x2048 -> 2x2x2048
    
        # Output layer
        X = X.reshape(X.shape[0], -1)
        X = torch.nn.Linear(in_features=X.shape[1], out_features=self.classes)
        X = torch.nn.Softmax(X)
    
        return X

next script:

NNet = ResNet50()

device = torch.device('cuda:0' if torch.cuda.is_available else 'cpu')
NNet = NNet.to(device)

loss = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(NNet.parameters(), lr = 0.001)

random.seed(k)
np.random.seed(k)
torch.manual_seed(k)
torch.cuda.manual_seed(k)
torch.backends.cudnn.deterministic = True

Returned error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-18-71ee8a51c6b2> in <module>()
      5 
      6 loss = torch.nn.CrossEntropyLoss()
----> 7 optimizer = torch.optim.Adam(NNet.parameters(), lr = 0.001)
      8 
      9 random.seed(k)

1 frames
/usr/local/lib/python3.6/dist-packages/torch/optim/optimizer.py in __init__(self, params, defaults)
     44         param_groups = list(params)
     45         if len(param_groups) == 0:
---> 46             raise ValueError("optimizer got an empty parameter list")
     47         if not isinstance(param_groups[0], dict):
     48             param_groups = [{'params': param_groups}]

ValueError: optimizer got an empty parameter list

Answer 1

Your class does not have any parameters, so .parameters() will give you an empty list.

You have to actually create the individual layers and store them in variables.

Right now all you do is call

X = torch.nn.Conv2d(in_channels=X.shape[0], out_channels=F1, kernel_size=1, stride=1, padding=0)(X)

Which creates an temporary Conv2d object, calls the forward function of that object and then the object is lost, since only the output of the forward is saved in x .

The correct thing to do is to either define your layers in the __init__() or a function which you call in the init.

So correct thing to do

 def __init__(self, input_shape = (3, 96, 96), classes = 10):
    super(ResNet50, self).__init__()    
    """
    Implementation of the popular ResNet50 the following architecture:
    Conv2d -> BatchNorm -> ReLU -> MaxPool -> ConvBlock - > IdBlock*2 - > convBlock -> IdBlock*3 -> ConvBlock -> IdBlock*5 -> ConvBlock -> IdBlock*2 -> AvgPool -> FCLayer

    Arguments:
    input_shape -- shape of the image of the dataset
    classes -- integer, number of classes    
    """
    self._conv_1 = torch.nn.Conv2d(in_channels=X.shape[0], out_channels=F1, kernel_size=1, stride=1, padding=0)
    self._bn_1 = torch.nn.BatchNorm2d(num_features=F1)

    ...

    self.input_shape = input_shape
    self.classes = classes
    self.relu = torch.nn.ReLU()

and later in your forward or a function called by forward you can do

def forward(self, X):
    """
    Forward propogation by the following architecture:
    Conv2d -> BatchNorm -> ReLU -> MaxPool -> ConvBlock - > IdBlock*2 - > convBlock -> IdBlock*3 -> ConvBlock -> IdBlock*5 -> ConvBlock -> IdBlock*2 -> AvgPool -> FCLayer

    Arguments:
    X -- input data for Network that needed to be propogated

    Returns:
    X -- output of the ResNet50, that propogated through it
    """

    # # Define the input as a tensor with shape self.input_shape
    # X = torch.zeros_like(self.input_shape)

    x = self.relu(self._bn_1(self._conv_1(x)))

    return X

So you have to do it along these lines. Create your layers and save them in variables and later use the variables in the forward.

For further reference and help refer to the official tutorial https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html