Tensorflow Keras无法在初始时从检查点文件正确恢复训练

Question

I am loading a keras model in tensorflow to resume training. I want to continue training from the epoch I stopped at so that epoch numbers are unique and I can keep track of the number of epochs. The model is loaded from a checkpoint file created by a callback that saves the highest accuracy. When I resume training in model.fit(), I set the "initial epoch" to be 52 and set "epoch" to 52+5. However, it starts training from epoch 1/57 instead of 53/57 and will keep going up to 57 even though I only want 5 epochs. Am I loading something wrongly? Training resumes as 'normal' and accuracy is where I left off, but the epoch numbers do not continue from where I want, and keep restarting from 1.

I have tried removing the checkpoint callback initialisation when loading form the checkpoint file, but that generates a name error as the "callbacks list" is not defined.

model = load_model('my_model.hdf5')
checkpoint = ModelCheckpoint(cp_filepath, monitor='acc', 
verbose=1, save_best_only=True, mode='max')
callbacks_list = [checkpoint]

bs=32 #batch size
epoch count=52
cur_epochs=5
model.fit(
    training_set,
    steps_per_epoch=len(training_set)//bs,
    inital_epoch=epoch_count,
    epochs=cur_epochs+epoch_count,
    validation_data=test_set,
    validation_steps=len(test_set)//bs,
    callbacks=callbacks_list, 
    shuffle=True,
    verbose=1
    )

I expect to see epoch 53/57 and 5 epochs of training when resuming from a saved file. I get epoch 1/57 and 57 epochs of training

Answer 1

I noticed that you forgot to put an underscore in epoch_count. That might be what is causing it.

Answer 2

Had the same issue, To solve it I modified the ModelCheckpoint (Callback) class.

I added and saved a dedicated tensorflow checkpoint for epoch, in the on_epoch_begin callback function.

The network doesn't store its training progress with respect to training data - this is not part of its state, because at any point you could decide to change what data set to feed it.

class EpochModelCheckpoint(tf.keras.callbacks.ModelCheckpoint):

    def __init__(self,filepath, monitor='val_loss', verbose=1, 
                 save_best_only=True, save_weights_only=True, 
                 mode='auto', ):

        super(EpochModelCheckpoint, self).__init__(filepath=filepath,monitor=monitor,
             verbose=verbose,save_best_only=save_best_only,
             save_weights_only=save_weights_only, mode=mode)

        self.ckpt = tf.train.Checkpoint(completed_epochs=tf.Variable(0,trainable=False,dtype='int32'))
        ckpt_dir = f'{os.path.dirname(filepath)}/tf_ckpts'
        self.manager = tf.train.CheckpointManager(self.ckpt, ckpt_dir, max_to_keep=3)

    def on_epoch_begin(self,epoch,logs=None):        
        self.ckpt.completed_epochs.assign(epoch)
        self.manager.save()
        print( f"Epoch checkpoint {self.ckpt.completed_epochs.numpy()}  saved to: {self.manager.latest_checkpoint}" ) 
        print(logs)

def callbacks(checkpoint_dir, model_name):

    best_model = os.path.join(checkpoint_dir, '{}_best.hdf5'.format(model_name))
    save_best = EpochModelCheckpoint( best_model  )
    return [ save_best ]

def train():

    ...

    model = get_compiled_model()
    checkpoint_dir = "./checkpoint_dir"
    model_name = "my_model"
    # Init checkpoint value
    ckpt = tf.train.Checkpoint(completed_epochs=tf.Variable(0,trainable=False,dtype='int32'))
    manager = tf.train.CheckpointManager(ckpt, f'{checkpoint_dir}/tf_ckpts', max_to_keep=3)    

    best_weights = os.path.join(checkpoint_dir, f'{model_name}_best.hdf5') 
    if os.path.exists(best_weights):
        print(f'Loading model {best_weights}')
        model.load_weights(best_weights)

        # Restore last Epoch
        ckpt.restore(manager.latest_checkpoint)
        if manager.latest_checkpoint:
            print(f"Restored epoch ckpt from {manager.latest_checkpoint}, value is ",ckpt.completed_epochs.numpy())
        else:
            print("Initializing from scratch.")

     ...
    # Set initial_epoch in the model fit to last seen Epoch
    completed_epochs=ckpt.completed_epochs.numpy()
    history = model.fit(
        x=train_ds,
        epochs=cfg.epochs,
        steps_per_epoch=cfg.steps,
        callbacks=callbacks( checkpoint_dir, model_name ),        
        validation_data=val_ds,
        validation_steps=cfg.val_steps,
        initial_epoch=completed_epochs )