在Keras自定义层中连接形状（None，m）的多个LSTM输出

Question

I'm trying to create a custom loss function using Keras custom layer template. I'm unable to perform calculations similar to what mentioned in the following paper (page 4, equation 5): https://web.stanford.edu/class/archive/cs/cs224n/cs224n.1174/reports/2748045.pdf

class CustomLoss(Layer):
    def __init__(self, **kwargs):
        self.result = None
        super(CustomLoss, self).__init__(**kwargs)

    def build(self, input_shape):
#         shape_ = 
        self.weight = self.add_weight(name='trainable_weight', shape=(input_shape[1], 4), initializer='glorot_uniform',
                                      trainable=True)
        self.bias = self.add_weight(name='trainable_bias', shape=(4, ), initializer='zeros', trainable=True)
        super(CustomLoss, self).build(input_shape)

    def call(self, input_vec, **kwargs):
        v1 = input_vec[0] # This is of shape (None, m)
        v2 = input_vec[1] # This is of shape (None, m)
        v3 = K.square(input_vec[0] - input_vec[1])
        v4 = K.dot(input_vec[0], input_vec[1])
        vec = concatenate([v1, v2, v3, v4])
        self.result = keras.layer.Softmax(keras.layers.ReLU(vec) * self.weight + self.bias)
        return self.result

    def compute_output_shape(self, input_shape):
        return K.int_shape(self.result)

I'm receiving following error:

TypeError: '>' not supported between instances of 'tuple' and 'float'

Edit1

num = 75
EMB_DIM = 300
SEN_LEN = 20

def base_network(_input):
    embd = embedding_layer(V_SIZE, EMB_DIM, SEN_LEN, embedding_matrix(_input)
    x = Bidirectional(lstm(num), merge_mode='concat')(embd)
    x = Bidirectional(lstm(num), merge_mode='concat')(x)
    x = Dropout(0.2)(x)
    y = TimeDistributed(Dense(1, activation='tanh'))(x)
    y = Flatten()(y)
    y = Activation('softmax')(y)
    y = RepeatVector(num * 2)(y)
    y = Permute([2, 1]) (y)
    z = multiply([x, y])
    z = Lambda(lambda xin: K.sum(xin, axis=1))(z)
    return z

inp1 = Input(shape=(SEN_LEN,), dtype='float32')
inp2 = Input(shape=(SEN_LEN,), dtype='float32')

s1 = base_network(inp1)
s2 = base_network(inp1)

sim_score = CustomLoss()([s1, s2])
output = concatenate([s1, s2 , sim_score])
d1 = Dense(2)(output)
sim = Dense(1, activation='sigmoid')(d1)
model = Model(inputs=[inp1, inp2], outputs=[sim])
model.compile(loss='mean_squared_error', optimizer=RMSprop())

Edit 2

TypeError                                 Traceback (most recent call last)
<ipython-input-97-fa4c18ab9e4e> in <module>
      7 s1 = base_network(input_2)
      8 
----> 9 sim_score = CustomLoss()([s1, s2])
     10 output = concatenate([s1, s2 , sim_score])
     11 d1 = Dense(2)(output)

~\.julia\conda\3\lib\site-packages\keras\engine\base_layer.py in __call__(self, inputs, **kwargs)
    429                                          'You can build it manually via: '
    430                                          '`layer.build(batch_input_shape)`')
--> 431                 self.build(unpack_singleton(input_shapes))
    432                 self.built = True
    433 

<ipython-input-96-2f2fb52e16d0> in build(self, input_shape)
    117 #         shape_ =
    118         self.weight = self.add_weight(name='trainable_weight', shape=(input_shape[1], 4, ), initializer='glorot_uniform',
--> 119                                       trainable=True)
    120         self.bias = self.add_weight(name='trainable_bias', shape=(4, ), initializer='zeros', trainable=True)
    121         super(CustomLoss, self).build(input_shape)

~\.julia\conda\3\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

~\.julia\conda\3\lib\site-packages\keras\engine\base_layer.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint)
    247         if dtype is None:
    248             dtype = K.floatx()
--> 249         weight = K.variable(initializer(shape),
    250                             dtype=dtype,
    251                             name=name,

~\.julia\conda\3\lib\site-packages\keras\initializers.py in __call__(self, shape, dtype)
    203         scale = self.scale
    204         if self.mode == 'fan_in':
--> 205             scale /= max(1., fan_in)
    206         elif self.mode == 'fan_out':
    207             scale /= max(1., fan_out)

TypeError: '>' not supported between instances of 'tuple' and 'float'

Edit 3 New Error:

---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
~\.julia\conda\3\lib\site-packages\tensorflow\python\framework\ops.py in _create_c_op(graph, node_def, inputs, control_inputs)
   1627   try:
-> 1628     c_op = c_api.TF_FinishOperation(op_desc)
   1629   except errors.InvalidArgumentError as e:

InvalidArgumentError: Dimensions must be equal, but are 2000 and 500 for 'custom_loss_1/MatMul' (op: 'MatMul') with input shapes: [?,2000], [500,4].

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-99-fa4c18ab9e4e> in <module>
      7 s1 = base_network(input_2)
      8 
----> 9 sim_score = CustomLoss()([s1, s2])
     10 output = concatenate([s1, s2 , sim_score])
     11 d1 = Dense(2)(output)

~\.julia\conda\3\lib\site-packages\keras\engine\base_layer.py in __call__(self, inputs, **kwargs)
    455             # Actually call the layer,
    456             # collecting output(s), mask(s), and shape(s).
--> 457             output = self.call(inputs, **kwargs)
    458             output_mask = self.compute_mask(inputs, previous_mask)
    459 

<ipython-input-98-23434db31b00> in call(self, x, **kwargs)
    127         vec = concatenate([v1, v2, v3, v4])
    128 #         vec = K.Flatten(vec)
--> 129         self.result = keras.layer.Softmax(keras.layers.ReLU(vec) * self.weight + self.bias)
    130         return self.result
    131 

~\.julia\conda\3\lib\site-packages\keras\backend\tensorflow_backend.py in dot(x, y)
   1083         out = tf.sparse_tensor_dense_matmul(x, y)
   1084     else:
-> 1085         out = tf.matmul(x, y)
   1086     return out
   1087 

~\.julia\conda\3\lib\site-packages\tensorflow\python\ops\math_ops.py in matmul(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name)
   2055     else:
   2056       return gen_math_ops.mat_mul(
-> 2057           a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
   2058 
   2059 

~\.julia\conda\3\lib\site-packages\tensorflow\python\ops\gen_math_ops.py in mat_mul(a, b, transpose_a, transpose_b, name)
   4855     _, _, _op = _op_def_lib._apply_op_helper(
   4856         "MatMul", a=a, b=b, transpose_a=transpose_a, transpose_b=transpose_b,
-> 4857         name=name)
   4858     _result = _op.outputs[:]
   4859     _inputs_flat = _op.inputs

~\.julia\conda\3\lib\site-packages\tensorflow\python\framework\op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
    785         op = g.create_op(op_type_name, inputs, output_types, name=scope,
    786                          input_types=input_types, attrs=attr_protos,
--> 787                          op_def=op_def)
    788       return output_structure, op_def.is_stateful, op
    789 

~\.julia\conda\3\lib\site-packages\tensorflow\python\util\deprecation.py in new_func(*args, **kwargs)
    486                 'in a future version' if date is None else ('after %s' % date),
    487                 instructions)
--> 488       return func(*args, **kwargs)
    489     return tf_decorator.make_decorator(func, new_func, 'deprecated',
    490                                        _add_deprecated_arg_notice_to_docstring(

~\.julia\conda\3\lib\site-packages\tensorflow\python\framework\ops.py in create_op(***failed resolving arguments***)
   3272           input_types=input_types,
   3273           original_op=self._default_original_op,
-> 3274           op_def=op_def)
   3275       self._create_op_helper(ret, compute_device=compute_device)
   3276     return ret

~\.julia\conda\3\lib\site-packages\tensorflow\python\framework\ops.py in __init__(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)
   1790           op_def, inputs, node_def.attr)
   1791       self._c_op = _create_c_op(self._graph, node_def, grouped_inputs,
-> 1792                                 control_input_ops)
   1793 
   1794     # Initialize self._outputs.

~\.julia\conda\3\lib\site-packages\tensorflow\python\framework\ops.py in _create_c_op(graph, node_def, inputs, control_inputs)
   1629   except errors.InvalidArgumentError as e:
   1630     # Convert to ValueError for backwards compatibility.
-> 1631     raise ValueError(str(e))
   1632 
   1633   return c_op

ValueError: Dimensions must be equal, but are 2000 and 500 for 'custom_loss_1/MatMul' (op: 'MatMul') with input shapes: [?,2000], [500,4].

Answer 1

This line:

self.result = keras.layer.Softmax(keras.layers.ReLU(vec) * self.weight + self.bias)

You can't use a layer like that. Instead use the equivalent backend functions:

self.result = K.softmax(K.bias_add(K.dot(K.relu(vec), self.weight), self.bias, data_format='channels_last'))

Note: I didn't read the paper, so also make sure you want to apply the relu on the concatenation result, and not after the dot product with weights and adding bias.

Update : there is also anther problem which is caused with the shape of weights you are creating. Since your layer gets two input tensors, then the input_shape argument of build method would be a list of two tuples, ie one input shape for corresponding to each input tensor. Therefore, when creating the weights, instead of writing shape=(input_shape[1], 4) you need to write shape=(input_shape[0][1], 4) .

Update 2 : vec has a shape of (?, 2000) but the weight has a shape of (500, 4) and therefore they cannot be multiplied. You probably want to adjust the shape of weight accordingly: use shape=(input_shape[0][1] * 4, 4) instead.