Reassignment of weights in tensorflow 2/keras

Question

I'm currently testing some modified versions of dropout in Keras and one of them involves adjusting the weights during the training of a customized dense layer. I however have not been able to run it without error yet. I suspect is has something to do with eager execution but I'm not sure.

class Linear(keras.layers.Layer):
def __init__(self, units, **kwargs):
    super(Linear, self).__init__(**kwargs)
    self.units = units

def build(self, input_shape):
    self.w = self.add_weight(
        shape=(input_shape[-1], self.units),
        initializer="random_normal",
        trainable=True,
    )
    self.b = self.add_weight(
        shape=(self.units,), initializer="random_normal", trainable=True
    )

def call(self, inputs, training=False):
    prob = 0.0/10
    if training:
        w = np.matrix(self.w)
        # w = self.w
        shape = w.shape
        size = shape[0] * shape[1]

        arr = np.random.choice([0,1], size=size, p=[prob, 1 - prob]) #random array of 1's and 0's
        arr = arr.reshape(shape) #reshape it to same dimensions as weights
        new_weights = np.multiply(arr, w) #element wise multiplication
        self.w = new_weights
    return tf.matmul(inputs, self.w) + self.b

model = models.Sequential()

model.add(layers.Conv2D(32, (3, 3), activation='relu', padding='same'))
model.add(layers.MaxPooling2D())

model.add(layers.Conv2D(64, (3, 3), activation='relu', padding='same'))
model.add(layers.MaxPooling2D())

model.add(layers.Conv2D(128, (3, 3), activation='relu',padding='same'))
model.add(layers.MaxPooling2D())

model.add(layers.Conv2D(4, (3, 3), activation='relu',padding='same'))
model.add(layers.MaxPooling2D())

model.add(layers.Flatten())
model.add(Linear(3)) #Custom layer
model.add(layers.Dense(10, activation='softmax'))

model.compile(loss = 'CategoricalCrossentropy',
              optimizer = 'adam',
              metrics=['accuracy'])

epochs = 1
history = model.fit(train_dataset, validation_data=validation_dataset, epochs=epochs)

Error: TypeError: Expected binary or unicode string, got <tf.Tensor 'sequential_3/linear_3/mul:0' shape=(4, 3) dtype=float32>

Answer 1

self.w has to be tensorflow.Variable . However after multiplication in call() it becomes tensorflow.Tensor . Just find another way to do the same thing in call() Try this code:

    def call(self, inputs, training=False):
        prob = 0.0/10
        if training:
            w = np.matrix(self.w)
            shape = w.shape
            size = shape[0] * shape[1]

            arr = np.random.choice([0,1], size=size, p=[prob, 1 - prob]) #random array of 1's and 0's
            arr = arr.reshape(shape) #reshape it to same dimensions as weights
# CHANGED 3 LINES BELOW:
            arr = tf.convert_to_tensor(arr, dtype=tf.float32)
            new_weights = tf.multiply(arr, self.w)
            self.w.assign(new_weights)  # Assign preserves tf.Variable

        return tf.matmul(inputs, self.w) + self.b