How to implement two layers of Keras conv1d in numpy?

Question

I have two conv1d layers in Keras that I'm trying to replicate using numpy. I have the first layer working correctly, but I'm having difficulty figuring out the second layer. The first conv1d layer takes an input_shape=(100,1), and outputs a shape of (None, 9, 16). The second conv1d layer outputs a shape of (None, 1, 16) in Keras. Here are my Keras layers:

model.add(Conv1D(16, 12,strides=12, activation=None, padding='same',input_shape=(100,1)))
model.add(Conv1D(16, 12,strides=12, activation=None, padding='same'))

and here is my conv1d code using numpy:

    def unfold(self, x, kernel_size, stride):  # WORK IN PROGRESS
        unfolded_x = np.array([[c for c in x[i*stride:i*stride+kernel_size]] for i in range(int(len(x)/stride))])

        unfolded_x = np.swapaxes(unfolded_x, 0,1)   
        unfolded_x = unfolded_x[np.newaxis, 0:, 0: ]
        return unfolded_x

    def conv1d_layer(self, input_data, weights, biases, kernel_size=1, stride=1, padding='same'):
        # Apply padding to input data
        in_width = len(input_data)
        if padding == 'same':

            if in_width % stride == 0:
                pad_along_width = max(kernel_size - stride, 0)
            else:
                pad_along_width = max(kernel_size - (in_width % stride), 0)

            pad_left = pad_along_width // 2
            pad_right = pad_along_width - pad_left
            input_data = np.concatenate([np.zeros(pad_left), input_data, np.zeros(pad_right)])
        

        # Take the dot product of input_data and weights (or unfolded input data)
        if kernel_size == 1 and stride == 1: #this is for dense layer
            y = np.dot(input_data, weights)
        else:
            #Create sliding window (unfolded) and perform dot product
            unfolded_input = self.unfold(input_data, kernel_size=kernel_size, stride=stride)

            y = np.tensordot(unfolded_input, weights, axes=([1, 0], [0, 1]))

        out = y + biases
        return out        

and my calls to the conv1d layer:

out = conv1d_layer(input, weights['W_conv1d'], weights['b_conv1d'], kernel_size=12, stride=12)
conv1d_layer2 = conv1d_layer(out, weights['W_conv1d'], weights['b_conv1d'], kernel_size=12, stride=12)

Obviously the second layer won't work because it's set up to take in (100,1), so how would I adjust my code to take in the first layer output of (9,16), given 'same' padding, kernel_size=12 and stride=12?

Thank you

Answer 1

I think this should do the trick:

import numpy as np

def pad(x, kernel_size, stride):
    """'same' padding"""
    seq_len, channels = x.shape
    if seq_len % stride == 0:
        pad_width = max(kernel_size - stride, 0)
    else:
        pad_width = max(kernel_size - (seq_len % stride), 0)
    pad_left = pad_width // 2
    pad_right = pad_width - pad_left
    return np.concatenate([np.zeros((pad_left, channels)), x, np.zeros((pad_right, channels))])

def unfold(x, kernel_size, stride):
    return np.stack([[c for c in x[i*stride:i*stride+kernel_size]] for i in range(len(x)//stride)])

def conv1d(x, weight, bias, stride):
    kernel_size = weight.shape[0]
    out = pad(x, kernel_size, stride)
    out = unfold(out, kernel_size, stride)
    # l = sequence, k = kernel, c = input channels, o = output channels
    # you can use np.tensordot here, but I find einsum is more readable, though be less performant
    out = np.einsum("lkc,kco->lo", out, weight) + bias
    return out

# input sequence
x = np.random.randn(100, 1)

# weights are shape [kernel size, input channels, output channels]
# biases are shape [output channels]

w1 = np.random.randn(12, 1, 16)
b1 = np.random.randn(16)
w2 = np.random.randn(12, 16, 16)
b2 = np.random.randn(16)

print("input", x.shape)
out = conv1d(x, w1, b1, stride=12)
print("first conv", out.shape)
out = conv1d(out, w2, b2, stride=12)
print("second conv", out.shape)

input (100, 1)
first conv (9, 16)
second conv (1, 16)

I think the issue you were running into is regarding the extra channel dimension. - Teddy

Note:

If you want a faster version of unfold you can leverage numpy.lib.stride_tricks.as_strided , for a vectorized approach:

from numpy.lib.stride_tricks import as_strided

def unfold(x, kernel_size, stride):
    return as_strided(
        x,
        shape=(x.shape[0] // stride, kernel_size, x.shape[1]),
    )

There can be some caveats with as_strided , checkout the documentation for more.