base.py 文件源码

def _clip_gradients_fn(self, grads_and_vars):
        """Clips gradients by global norm."""
        gradients, variables = zip(*grads_and_vars)
        self._grads_and_vars = grads_and_vars

        if self._clip_gradients > 0.0:
            clipped_gradients, _ = tf.clip_by_global_norm(
                t_list=gradients, clip_norm=self._clip_gradients)
            grads_and_vars = list(zip(clipped_gradients, variables))
        if self._clip_embed_gradients > 0.0:
            clipped_gradients = []
            variables = []
            for gradient, variable in grads_and_vars:
                if "embedding" in variable.name or "Embedding" in variable.name:
                    tmp = tf.clip_by_norm(t=gradient.values, clip_norm=self._clip_embed_gradients)
                    gradient = tf.IndexedSlices(tmp, gradient.indices, gradient.dense_shape)
                clipped_gradients.append(gradient)
                variables.append(variable)
            grads_and_vars = list(zip(clipped_gradients, variables))
        return grads_and_vars