def _init_weight(self):
init.xavier_uniform(self.transitions)
self.transitions.data[START, :].fill_(-10000.)
self.transitions.data[:, STOP].fill_(-10000.)
python类xavier_uniform()的实例源码
def _init_weight(self, scope=1.):
init.xavier_uniform(self.char_ebd.weight)
def _init_weights(self, scope=1.):
self.embedded_chars_left.weight.data.uniform_(-scope, scope)
self.embedded_chars_right.weight.data.uniform_(-scope, scope)
init.xavier_uniform(self.simi_weight)
init.xavier_uniform(self.out_lr.weight)
init.xavier_uniform(self.logistic.weight)
def _init_weights(self, scope=0.25):
self.lookup_table.weight.data.uniform_(-scope, scope)
init.xavier_uniform(self.logistic.weight)
def _init_weights(self, scope=0.25):
self.lookup_table.weight.data.uniform_(-scope, scope)
init.xavier_uniform(self.logistic.weight)
def xavier(param):
init.xavier_uniform(param)
def xavier(param):
init.xavier_uniform(param)
def xavier(param):
init.xavier_uniform(param)
def glorot_weight_zero_bias(model):
"""
Initalize parameters of all modules
by initializing weights with glorot uniform/xavier initialization,
and setting biases to zero.
Weights from batch norm layers are set to 1.
Parameters
----------
model: Module
"""
for module in model.modules():
if hasattr(module, 'weight'):
if not ('BatchNorm' in module.__class__.__name__):
init.xavier_uniform(module.weight, gain=1)
else:
init.constant(module.weight, 1)
if hasattr(module, 'bias'):
if module.bias is not None:
init.constant(module.bias, 0)
def conv_init(m):
classname = m.__class__.__name__
if classname.find('Conv') != -1:
init.xavier_uniform(m.weight, gain=np.sqrt(2))
init.constant(m.bias, 0)
elif classname.find('BatchNorm') != -1:
init.constant(m.weight, 1)
init.constant(m.bias, 0)
def weights_init(self,module):
for m in module.modules():
if isinstance(m, nn.Conv2d):
init.xavier_uniform(m.weight, gain=np.sqrt(2))
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def weights_init(self,module):
for m in module.modules():
if isinstance(m, nn.Conv2d):
init.xavier_uniform(m.weight, gain=np.sqrt(2))
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Linear') != -1:
init.xavier_uniform(m.weight.data)
m.bias.data.fill_(0)
def __init__(self, input_dim, dropout=0, softplus_boost=1.0):
super(ProposalBeta, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 2)
self.drop = nn.Dropout(dropout)
self.softplus_boost = softplus_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, output_dim, dropout=0, softplus_boost=1.0):
super(ProposalMultivariateNormal, self).__init__()
self.mean_lin1 = nn.Linear(input_dim, input_dim)
self.mean_drop = nn.Dropout(dropout)
self.mean_lin2 = nn.Linear(input_dim, output_dim)
self.vars_lin1 = nn.Linear(input_dim, input_dim)
self.vars_drop = nn.Dropout(dropout)
self.vars_lin2 = nn.Linear(input_dim, output_dim)
self.softplus_boost = softplus_boost
init.xavier_uniform(self.mean_lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.mean_lin2.weight)
init.xavier_uniform(self.vars_lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.vars_lin2.weight)
def __init__(self, input_example_non_batch, output_dim, dropout=0):
super(ObserveEmbeddingFC, self).__init__()
self.input_dim = input_example_non_batch.nelement()
self.lin1 = nn.Linear(self.input_dim, output_dim)
self.lin2 = nn.Linear(output_dim, output_dim)
self.drop = nn.Dropout(dropout)
init.xavier_uniform(self.lin1.weight, gain=np.sqrt(2.0))
init.xavier_uniform(self.lin2.weight, gain=np.sqrt(2.0))
def test_xavier_uniform_errors_on_inputs_smaller_than_2d(self):
for as_variable in [True, False]:
for dims in [0, 1]:
tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
with self.assertRaises(ValueError):
init.xavier_uniform(tensor)
def test_xavier_uniform(self):
for as_variable in [True, False]:
for use_gain in [True, False]:
for dims in [2, 4]:
input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
as_variable=as_variable)
gain = 1
if use_gain:
gain = self._random_float(0.1, 2)
init.xavier_uniform(input_tensor, gain=gain)
else:
init.xavier_uniform(input_tensor)
if as_variable:
input_tensor = input_tensor.data
fan_in = input_tensor.size(1)
fan_out = input_tensor.size(0)
if input_tensor.dim() > 2:
fan_in *= input_tensor[0, 0].numel()
fan_out *= input_tensor[0, 0].numel()
expected_std = gain * math.sqrt(2.0 / (fan_in + fan_out))
bounds = expected_std * math.sqrt(3)
assert self._is_uniform(input_tensor, -bounds, bounds)
def xavier_uniform(w, gain=1):
return nn.xavier_uniform(w, gain=gain)
def conv_fc_init(layer,
weight_init=init.xavier_uniform,
bias_init=zero_init):
"""
Initialize a layer's filter weights by xavier and bias weights to zero
The layer can be either nn.ConvNd or nn.Linear
"""
if isinstance(layer, (list, nn.ModuleList)):
return type(layer)([conv_fc_init(l, weight_init=weight_init, bias_init=bias_init)
for l in layer])
assert is_conv_layer(layer) or isinstance(layer, nn.Linear)
weight_init(layer.weight)
bias_init(layer.bias)
return layer
def xavier(param):
init.xavier_uniform(param)
def test_xavier_uniform_errors_on_inputs_smaller_than_2d(self):
for as_variable in [True, False]:
for dims in [0, 1]:
tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
with self.assertRaises(ValueError):
init.xavier_uniform(tensor)
def test_xavier_uniform(self):
for as_variable in [True, False]:
for use_gain in [True, False]:
for dims in [2, 4]:
input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
as_variable=as_variable)
gain = 1
if use_gain:
gain = self._random_float(0.1, 2)
init.xavier_uniform(input_tensor, gain=gain)
else:
init.xavier_uniform(input_tensor)
if as_variable:
input_tensor = input_tensor.data
fan_in = input_tensor.size(1)
fan_out = input_tensor.size(0)
if input_tensor.dim() > 2:
fan_in *= input_tensor[0, 0].numel()
fan_out *= input_tensor[0, 0].numel()
expected_std = gain * math.sqrt(2.0 / (fan_in + fan_out))
bounds = expected_std * math.sqrt(3)
assert self._is_uniform(input_tensor, -bounds, bounds)
def weights_init(self,module):
for m in module.modules():
if isinstance(m, nn.Conv2d):
init.xavier_uniform(m.weight, gain=np.sqrt(2))
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def test_xavier_uniform_errors_on_inputs_smaller_than_2d(self):
for as_variable in [True, False]:
for dims in [0, 1]:
tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
with self.assertRaises(ValueError):
init.xavier_uniform(tensor)
def test_xavier_uniform(self):
for as_variable in [True, False]:
for use_gain in [True, False]:
for dims in [2, 4]:
input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
as_variable=as_variable)
gain = 1
if use_gain:
gain = self._random_float(0.1, 2)
init.xavier_uniform(input_tensor, gain=gain)
else:
init.xavier_uniform(input_tensor)
if as_variable:
input_tensor = input_tensor.data
fan_in = input_tensor.size(1)
fan_out = input_tensor.size(0)
if input_tensor.dim() > 2:
fan_in *= input_tensor[0, 0].numel()
fan_out *= input_tensor[0, 0].numel()
expected_std = gain * math.sqrt(2.0 / (fan_in + fan_out))
bounds = expected_std * math.sqrt(3)
assert self._is_uniform(input_tensor, -bounds, bounds)
def init_param(self, param):
if len(param.size()) < 2:
init.uniform(param)
else:
init.xavier_uniform(param)
def init_param(self, param):
if len(param.size()) < 2:
init.uniform(param)
else:
init.xavier_uniform(param)
def test_xavier_uniform_errors_on_inputs_smaller_than_2d(self):
for as_variable in [True, False]:
for dims in [0, 1]:
tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
with self.assertRaises(ValueError):
init.xavier_uniform(tensor)
def test_xavier_uniform(self):
for as_variable in [True, False]:
for use_gain in [True, False]:
for dims in [2, 4]:
input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
as_variable=as_variable)
gain = 1
if use_gain:
gain = self._random_float(0.1, 2)
init.xavier_uniform(input_tensor, gain=gain)
else:
init.xavier_uniform(input_tensor)
if as_variable:
input_tensor = input_tensor.data
fan_in = input_tensor.size(1)
fan_out = input_tensor.size(0)
if input_tensor.dim() > 2:
fan_in *= input_tensor[0, 0].numel()
fan_out *= input_tensor[0, 0].numel()
expected_std = gain * math.sqrt(2.0 / (fan_in + fan_out))
bounds = expected_std * math.sqrt(3)
assert self._is_uniform(input_tensor, -bounds, bounds)
