gradient_descent.py 文件源码

def get_adagrad(learning_rate=0.5):
    """
    Adaptive Subgradient Methods for Online Learning and Stochastic Optimization
    John Duchi, Elad Hazan and Yoram Singer, Journal of Machine Learning Research 12 (2011) 2121-2159
    http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf
    """
    sum_square_gradient = None

    def adagrad(gradient):
        nonlocal sum_square_gradient

        if sum_square_gradient is None:
            sum_square_gradient = np.ones_like(gradient)
        sum_square_gradient += gradient ** 2
        return learning_rate / np.sqrt(sum_square_gradient)

    return adagrad