Currently I am working on speaker Diarization where I have to perform LSTM. I want to make custom optimizer in keras. For this i reimplemented SMORMS3 in custom way, i mean i define class for LSTM. some part of the code: I take code from https://gist.github.com/hbredin/89b2249504a62712441e7ffec7de9518
from keras.legacy import interfaces
from keras.optimizers import Optimizer
import keras.backend as K
class SMORMS3(Optimizer):
def __init__(self, lr=0.001, epsilon=1e-16, decay=0.,
**kwargs):
super(SMORMS3, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.lr = K.variable(lr, name='lr')
self.decay = K.variable(decay, name='decay')
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
shapes = [K.get_variable_shape(p) for p in params]
ms = [K.zeros(shape) for shape in shapes]
vs = [K.zeros(shape) for shape in shapes]
mems = [K.zeros(shape) for shape in shapes]
self.weights = [self.iterations] + ms + vs + mems
self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr *= (1. / (1. + self.decay * K.cast(self.iterations, K.dtype(self.decay))))
for p, g, m, v, mem in zip(params, grads, ms, vs, mems):
r = 1. / (1. + mem)
new_m = (1. - r) * m + r * g
new_v = (1. - r) * v + r * K.square(g)
denoise = K.square(new_m) / (new_v + self.epsilon)
new_p = p - g * K.minimum(lr, denoise) / (K.sqrt(new_v) + self.epsilon)
new_mem = 1. + mem * (1. - denoise)
self.updates.append(K.update(m, new_m))
self.updates.append(K.update(v, new_v))
self.updates.append(K.update(mem, new_mem))
# Apply constraints.
if getattr(p, 'constraint', None) is not None:
new_p = p.constraint(new_p)
self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)),
'decay': float(K.get_value(self.decay)),
'epsilon': self.epsilon}
base_config = super(SMORMS3, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
and here is the some part of my code for lstm:
from keras.layers.core import Dense
from keras.models import Sequential
from keras.layers import Bidirectional, TimeDistributed, Dropout
from keras.layers import LSTM
import numpy as np
import keras
def train_bilstm():
model = Sequential()
model.add(Bidirectional(LSTM(128, return_sequences=True)))
model.add(Dropout(0.3))
model.add(Bidirectional(LSTM(128, return_sequences=True)))
model.add(Dropout(0.3))
model.add(TimeDistributed(Dense(32)))
model.add(TimeDistributed(Dense(32)))
model.add(TimeDistributed(Dense(1, activation='sigmoid')))
model.build(input_shape=(None, 137, 35))
model.compile(loss=keras.losses.binary_crossentropy, optimizer=SMORMS3(0.001, 1e-16, 0.), metrics=['accuracy'])
model.summary()
print("subhash")
all_x, all_y = load_dataset()
print(all_y.shape, np.sum(all_y))
subsample_all_x = []
subsample_all_y = []
for index in range(all_y.shape[0]):
class_positive = sum(all_y[index])
if class_positive > 5:
subsample_all_x.append(all_x[index][np.newaxis, :, :])
subsample_all_y.append(all_y[index])
all_x = np.vstack(subsample_all_x)
all_y = np.vstack(subsample_all_y)
print(all_y.shape, np.sum(all_y))
all_y = all_y[:, :, np.newaxis]
indices = np.random.permutation(all_x.shape[0])
all_x_random = all_x[indices]
all_y_random = all_y[indices]
datasize = all_x_random.shape[0]
train_size = int(datasize*0.97)
train_x = all_x_random[0:train_size]
valid_x = all_x_random[train_size:]
train_y = all_y_random[0:train_size]
valid_y = all_y_random[train_size:]
print('train over')
my = model.fit(x=train_x, y=train_y, batch_size=256, epochs=50,
validation_data=(valid_x, valid_y), shuffle=True)
model.save('model_hindi_2.h5')
def save_model(model, json_model_file, h5_model_file):
# serialize model to JSON
model_json = model.to_json()
with open(json_model_file, "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights(h5_model_file)
print("Saved model to disk")
# model_name = 'speech_seg1'
# json_model_file = '/content/drive/My Drive/SRU/model_hindi_2'+'.json'
# h5_model_file = '/content/drive/My Drive/SRU/model_hindi_2'+'.h5'
# save_model(model, json_model_file, h5_model_file)
I got a error:
TypeError Traceback (most recent call last)
<ipython-input-133-e28aeaab5d03> in <module>
----> 1 train_bilstm()
<ipython-input-132-f20f9c93ac69> in train_bilstm()
20 model.build(input_shape=(None, 137, 35))
21
---> 22 model.compile(loss=keras.losses.binary_crossentropy, optimizer=SMORMS3(), metrics=['accuracy'])
23 model.summary()
24 print("subhash")
<ipython-input-127-f43b8a02704f> in __init__(self, lr, epsilon, decay, **kwargs)
20 def __init__(self, lr=0.001, epsilon=1e-16, decay=0.,
21 **kwargs):
---> 22 super(SMORMS3, self).__init__(**kwargs)
23 with K.name_scope(self.__class__.__name__):
24 self.lr = K.variable(lr, name='lr')
TypeError: __init__() missing 1 required positional argument: 'name'
When you create an instance of the Optimizer class, you have to give it a name. This is explained in the documentation for tf.keras.optimizers.Optimizer` .
In this case, modify this line to include a name:
super(SMORMS3, self).__init__(name="SMORMS3", **kwargs)
A more flexible option would be to modify your __init__
signature to include the name
variable with a default argument, this would allow you to overwrite the name if you want to, just as you can for the pre-defined optimizers:
def __init__(self, lr=0.001, epsilon=1e-16, decay=0., name="SMORMS3"
**kwargs):
super(SMORMS3, self).__init__(name=name, **kwargs)
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.