def _initialize_weights(self):
init.orthogonal(self.conv1.weight, init.calculate_gain('relu'))
init.orthogonal(self.conv2.weight, init.calculate_gain('relu'))
init.orthogonal(self.conv3.weight, init.calculate_gain('relu'))
init.orthogonal(self.conv4.weight)
python类calculate_gain()的实例源码
def __init__(self, input_dim, dropout=0, softplus_boost=1.0):
super(ProposalBeta, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 2)
self.drop = nn.Dropout(dropout)
self.softplus_boost = softplus_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, output_dim, dropout=0, softplus_boost=1.0):
super(ProposalMultivariateNormal, self).__init__()
self.mean_lin1 = nn.Linear(input_dim, input_dim)
self.mean_drop = nn.Dropout(dropout)
self.mean_lin2 = nn.Linear(input_dim, output_dim)
self.vars_lin1 = nn.Linear(input_dim, input_dim)
self.vars_drop = nn.Dropout(dropout)
self.vars_lin2 = nn.Linear(input_dim, output_dim)
self.softplus_boost = softplus_boost
init.xavier_uniform(self.mean_lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.mean_lin2.weight)
init.xavier_uniform(self.vars_lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.vars_lin2.weight)
def test_calculate_gain_linear(self):
for fn in ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose2d', 'conv_transpose2d', 'conv_transpose3d']:
gain = init.calculate_gain(fn)
self.assertEqual(gain, 1)
def __init__(self, input_dim, output_dim, dropout=0, softmax_boost=1.0):
super(ProposalUniformDiscrete, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, output_dim)
self.drop = nn.Dropout(dropout)
self.softmax_boost = softmax_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, dropout=0):
super(ProposalNormal, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 2)
self.drop = nn.Dropout(dropout)
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, dropout=0):
super(ProposalLaplace, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 2)
self.drop = nn.Dropout(dropout)
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, dropout=0, softmax_boost=1.0):
super(ProposalFlip, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 1)
self.drop = nn.Dropout(dropout)
self.softmax_boost = softmax_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, output_dim, dropout=0, softmax_boost=1.0):
super(ProposalDiscrete, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, output_dim)
self.drop = nn.Dropout(dropout)
self.softmax_boost = softmax_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
def __init__(self, input_dim, dropout=0, softplus_boost=1.0):
super(ProposalUniformContinuous, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 2)
self.drop = nn.Dropout(dropout)
self.softplus_boost = softplus_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, mixture_components=10, dropout=0):
super(ProposalUniformContinuousAlt, self).__init__()
self.mixture_components = mixture_components
self.output_dim = 3 * mixture_components
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, self.output_dim)
self.drop = nn.Dropout(dropout)
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def __init__(self, input_dim, dropout=0, softplus_boost=1.0):
super(ProposalGamma, self).__init__()
self.lin1 = nn.Linear(input_dim, input_dim)
self.lin2 = nn.Linear(input_dim, 2)
self.drop = nn.Dropout(dropout)
self.softplus_boost = softplus_boost
init.xavier_uniform(self.lin1.weight, gain=init.calculate_gain('relu'))
init.xavier_uniform(self.lin2.weight)
def test_calculate_gain_linear(self):
for fn in ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose2d', 'conv_transpose2d', 'conv_transpose3d']:
gain = init.calculate_gain(fn)
self.assertEqual(gain, 1)
def test_calculate_gain_nonlinear(self):
for fn in ['sigmoid', 'tanh', 'relu', 'leaky_relu']:
gain = init.calculate_gain(fn)
if fn == 'sigmoid':
self.assertEqual(gain, 1)
elif fn == 'tanh': # 5 / 3
self.assertEqual(gain, 1.6666666666666667)
elif fn == 'relu': # sqrt(2)
self.assertEqual(gain, 1.4142135623730951)
elif fn == 'leaky_relu': # sqrt(2 / 1 + slope^2))
self.assertEqual(gain, 1.4141428569978354)
def test_calculate_gain_leaky_relu(self):
for param in [None, 0, 0.01, 10]:
gain = init.calculate_gain('leaky_relu', param)
if param is None: # Default slope is 0.01
self.assertEqual(gain, 1.4141428569978354)
elif param == 0: # No slope = same gain as normal ReLU
self.assertEqual(gain, 1.4142135623730951)
elif param == 0.01:
self.assertEqual(gain, 1.4141428569978354)
elif param == 10:
self.assertEqual(gain, 0.14071950894605836)
def test_calculate_gain_leaky_relu_only_accepts_numbers(self):
for param in [True, [1], {'a': 'b'}]:
with self.assertRaises(ValueError):
init.calculate_gain('leaky_relu', param)
def test_calculate_gain_only_accepts_valid_nonlinearities(self):
for n in [2, 5, 25]:
# Generate random strings of lengths that definitely aren't supported
random_string = ''.join([random.choice(string.ascii_lowercase) for i in range(n)])
with self.assertRaises(ValueError):
init.calculate_gain(random_string)
def test_calculate_gain_linear(self):
for fn in ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose2d', 'conv_transpose2d', 'conv_transpose3d']:
gain = init.calculate_gain(fn)
self.assertEqual(gain, 1)
def test_calculate_gain_nonlinear(self):
for fn in ['sigmoid', 'tanh', 'relu', 'leaky_relu']:
gain = init.calculate_gain(fn)
if fn == 'sigmoid':
self.assertEqual(gain, 1)
elif fn == 'tanh': # 5 / 3
self.assertEqual(gain, 1.6666666666666667)
elif fn == 'relu': # sqrt(2)
self.assertEqual(gain, 1.4142135623730951)
elif fn == 'leaky_relu': # sqrt(2 / 1 + slope^2))
self.assertEqual(gain, 1.4141428569978354)
def test_calculate_gain_leaky_relu(self):
for param in [None, 0, 0.01, 10]:
gain = init.calculate_gain('leaky_relu', param)
if param is None: # Default slope is 0.01
self.assertEqual(gain, 1.4141428569978354)
elif param == 0: # No slope = same gain as normal ReLU
self.assertEqual(gain, 1.4142135623730951)
elif param == 0.01:
self.assertEqual(gain, 1.4141428569978354)
elif param == 10:
self.assertEqual(gain, 0.14071950894605836)
def test_calculate_gain_leaky_relu_only_accepts_numbers(self):
for param in [True, [1], {'a': 'b'}]:
with self.assertRaises(ValueError):
init.calculate_gain('leaky_relu', param)
def test_calculate_gain_only_accepts_valid_nonlinearities(self):
for n in [2, 5, 25]:
# Generate random strings of lengths that definitely aren't supported
random_string = ''.join([random.choice(string.ascii_lowercase) for i in range(n)])
with self.assertRaises(ValueError):
init.calculate_gain(random_string)
def test_calculate_gain_nonlinear(self):
for fn in ['sigmoid', 'tanh', 'relu', 'leaky_relu']:
gain = init.calculate_gain(fn)
if fn == 'sigmoid':
self.assertEqual(gain, 1)
elif fn == 'tanh': # 5 / 3
self.assertEqual(gain, 1.6666666666666667)
elif fn == 'relu': # sqrt(2)
self.assertEqual(gain, 1.4142135623730951)
elif fn == 'leaky_relu': # sqrt(2 / 1 + slope^2))
self.assertEqual(gain, 1.4141428569978354)
def test_calculate_gain_leaky_relu(self):
for param in [None, 0, 0.01, 10]:
gain = init.calculate_gain('leaky_relu', param)
if param is None: # Default slope is 0.01
self.assertEqual(gain, 1.4141428569978354)
elif param == 0: # No slope = same gain as normal ReLU
self.assertEqual(gain, 1.4142135623730951)
elif param == 0.01:
self.assertEqual(gain, 1.4141428569978354)
elif param == 10:
self.assertEqual(gain, 0.14071950894605836)
def test_calculate_gain_leaky_relu_only_accepts_numbers(self):
for param in [True, [1], {'a': 'b'}]:
with self.assertRaises(ValueError):
init.calculate_gain('leaky_relu', param)
def test_calculate_gain_only_accepts_valid_nonlinearities(self):
for n in [2, 5, 25]:
# Generate random strings of lengths that definitely aren't supported
random_string = ''.join([random.choice(string.ascii_lowercase) for i in range(n)])
with self.assertRaises(ValueError):
init.calculate_gain(random_string)
def initializationhelper(param, nltype):
c = 0.1
torchinit.uniform(param.weight, a=-c, b=c)
#torchinit.xavier_uniform(param.weight, gain=c*torchinit.calculate_gain(nltype))
c = 0.1
torchinit.uniform(param.bias, a=-c, b=c)
def test_calculate_gain_linear(self):
for fn in ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose2d', 'conv_transpose2d', 'conv_transpose3d']:
gain = init.calculate_gain(fn)
self.assertEqual(gain, 1)
def test_calculate_gain_nonlinear(self):
for fn in ['sigmoid', 'tanh', 'relu', 'leaky_relu']:
gain = init.calculate_gain(fn)
if fn == 'sigmoid':
self.assertEqual(gain, 1)
elif fn == 'tanh': # 5 / 3
self.assertEqual(gain, 1.6666666666666667)
elif fn == 'relu': # sqrt(2)
self.assertEqual(gain, 1.4142135623730951)
elif fn == 'leaky_relu': # sqrt(2 / 1 + slope^2))
self.assertEqual(gain, 1.4141428569978354)
def test_calculate_gain_leaky_relu(self):
for param in [None, 0, 0.01, 10]:
gain = init.calculate_gain('leaky_relu', param)
if param is None: # Default slope is 0.01
self.assertEqual(gain, 1.4141428569978354)
elif param == 0: # No slope = same gain as normal ReLU
self.assertEqual(gain, 1.4142135623730951)
elif param == 0.01:
self.assertEqual(gain, 1.4141428569978354)
elif param == 10:
self.assertEqual(gain, 0.14071950894605836)
