在 tensorflow keras 中创建熵作为自定义损失函数

Question

I am trying to create a custom loss function in tensorflow.keras; particularly, shannon's entropy. Here is the basic neural net structure

import tensorflow as tf
import numpy as np

import matplotlib.pyplot as plt

from scipy.stats import entropy
import numpy as np




mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train =x_train / 255.0





model = tf.keras.models.Sequential([

  tf.keras.layers.Flatten(input_shape=(28, 28, 1)),

  tf.keras.layers.Dense(128, activation=tf.nn.sigmoid),
  tf.keras.layers.Dense(10, activation=tf.nn.sigmoid)
])
model.compile(optimizer='sgd',
              loss=entropy_loss,
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=1,batch_size=512)

And I am trying 2 ways to calculate entropy, neither of which is working. The 1st way is to convert the y_true and y_pred to numpy, get the error, then calculate entropy using scipy's entropy measure. I am facing errors for converting to numpy. The 2nd way I am using tensorflow calculations, based on this:

how to calculate entropy on float numbers over a tensor in python keras

and still am facing errors there.

method1

def entropy_loss(y_true,y_pred):

    # Create a loss function that adds the MSE loss to the mean of all squared activations of a specific layer
   
    return tf.cast(entropy(y_pred.numpy() - y_true.numpy() , base=2))
   
    # Return a function
    #return loss

the 1st way has this error:

    <ipython-input-4-14c95bd6b1a3>:5 entropy_loss  *
        return tf.cast(entropy(y_pred.numpy() - y_true.numpy() , base=2))

    AttributeError: 'Tensor' object has no attribute 'numpy'

method2

def entropy_loss(y_true,y_pred):

    y_true=tf.cast(y_true, tf.float32)
    y_pred=tf.cast(y_pred, tf.float32)
    e=y_true-y_pred
    print(e)
    loss= entropy_1(e) 
    #return e
    # Return a function
    return loss
def entropy_1( x):
    def row_entropy(row):
        _, _, count = tf.unique_with_counts(row)
        prob = count / tf.reduce_sum(count)
        return -tf.reduce_sum(prob * tf.math.log(prob))

    value_ranges = [-10.0, 100.0]
    nbins = 50
    new_f_w_t = tf.histogram_fixed_width_bins(x, value_ranges, nbins)
    result = tf.map_fn(row_entropy, new_f_w_t,dtype=tf.float32)
    return result

This method has the following error:

    ValueError: Trying to read from list with wrong element dtype. List has type double but expected type float for '{{node entropy_loss/map/TensorArrayV2Stack/TensorListStack}} = TensorListStack[element_dtype=DT_FLOAT, num_elements=-1](entropy_loss/map/while:3, entropy_loss/map/TensorArrayV2Stack/Const)' with input shapes: [], [0].

Answer 1

Actually, you don't need to implement it. But, let's figure out.

Suppose, you have batch two, answers and predictions. You suspect, that tf entropy and entropy from scipy will give the same results.

a = np.array([[1], [2]], dtype=np.float)
b = np.array([[0.2, 0.7, 0.1], [0.2, 0.3, 0.5]], dtype=np.float)
at = tf.convert_to_tensor(a)
bt = tf.convert_to_tensor(b)
H = tf.keras.losses.sparse_categorical_crossentropy(a, b)
print(f"TF entropy: {H}")

a = [[0.2, 0.7, 0.1], [0.2, 0.3, 0.5]]
b = [[0, 1, 0], [0, 0, 1]]
H2 = entropy(b, a, axis=1)
print(f"Scipy entropy: {H2}")

Results:
TF entropy: [0.35667494 0.69314718]
Scipy entropy: [0.35667494 0.69314718]

Ok, let's implement it.

def my_entropy(y_true, y_pred):
    shape = tf.shape(y_pred)
    batch = shape[0]
    depth = tf.shape(y_pred)[1]
    y_true = tf.cast(y_true, tf.int32)
    y_true = tf.reshape(y_true, shape=(-1, batch))
    one_hot = tf.one_hot(y_true, depth=depth, dtype=tf.float32)
    y_pred = tf.cast(y_pred, dtype=tf.float32)
    div = tf.divide(one_hot, y_pred)
    div = tf.reduce_sum(div, axis=0)
    ind = tf.where(tf.greater(div, 0))
    values = tf.gather_nd(div, ind)
    h = tf.math.log(values)
    return h

Test:
H3 = my_entropy(at, bt)
My entropy: [0.35667497 0.6931472 ]

Now, you can use it as custom loss function as follow:

import tensorflow as tf

def my_entropy(y_true, y_pred):
    shape = tf.shape(y_pred)
    batch = shape[0]
    depth = tf.shape(y_pred)[1]
    y_true = tf.cast(y_true, tf.int32)
    y_true = tf.reshape(y_true, shape=(-1, batch))
    one_hot = tf.one_hot(y_true, depth=depth, dtype=tf.float32)
    y_pred = tf.cast(y_pred, dtype=tf.float32)
    div = tf.divide(one_hot, y_pred)
    div = tf.reduce_sum(div, axis=0)
    ind = tf.where(tf.greater(div, 0))
    values = tf.gather_nd(div, ind)
    h = tf.math.log(values)
    return h


if __name__ == '__main__':
    mnist = tf.keras.datasets.mnist

    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train / 255.0

    model = tf.keras.models.Sequential([

        tf.keras.layers.Flatten(input_shape=(28, 28, 1)),

        tf.keras.layers.Dense(128, activation=tf.nn.sigmoid),
        tf.keras.layers.Dense(10, activation=tf.nn.sigmoid)
    ])
    model.compile(optimizer='sgd',
                  loss=my_entropy,
                  metrics=['accuracy'])

    model.fit(x_train, y_train, epochs=5, batch_size=2)