Keskar, N. S., & Socher, R. (2017).
Improving generalization performance by switching from adam to sgd.
arXiv preprint arXiv:1712.07628.

model = Sequential()
...
model.compile(...)    


AdamToSGD_ = [AdamToSGD(after_training_with_Adam=SWATS(x=train_x, 
                                                       y=train_y,
                                                       ...))]
                                                       
                                                       
def SWATS(momentum=0.0,    # SGD optimizer arguments
          nesterov=False
          ...,
          loss='mse',      # compile arguments
          ...,
          x=None,          # model.fit statements
          y=None,
          **kwargs):
    """
    This user-defined function restarts training if condition 4 from 
    1712.07628 is satisfied in the callback.
    Define optimizer, compile it, and fit model again in one function.
    """
    lr = float(K.get_value(model.optimizer.lr))
    bias_corrected_exponential_avg = lr / (1. - K.get_value(model.optimizer.beta_2))
 
    if (K.abs(bias_corrected_exponential_avg - lr) < 1e-9) is not None:
        return
    else:
        SGD_optimizer = SGD(lr=bia_corrected_exponential_avg,
                            ...)
     
        model.compile(optimizer=SGD_optimizer,
                      ...)
                   
        print('\nNow switching to SGD...\n')
     
        model.fit(x=x,
                  y=y,
                  ...)
  
 
result = model.fit(train_x,
                   train_y,
                   callbacks=[AdamToSGD_, ...],
                   ...)