def R_weight_init(ms):
for m in ms.modules():
classname = m.__class__.__name__
if classname.find('Conv') != -1:
m.weight.data = init.kaiming_normal(m.weight.data)
elif classname.find('BatchNorm') != -1:
m.weight.data.normal_(1.0, 0.02)
m.bias.data.fill_(0)
elif classname.find('Linear') != -1:
m.weight.data = init.kaiming_normal(m.weight.data)
############################
# G network
###########################
# custom weights initialization called on netG
python类kaiming_normal()的实例源码
def reset_parameters(self):
if self.use_leaf_rnn:
init.kaiming_normal(self.leaf_rnn_cell.weight_ih.data)
init.orthogonal(self.leaf_rnn_cell.weight_hh.data)
init.constant(self.leaf_rnn_cell.bias_ih.data, val=0)
init.constant(self.leaf_rnn_cell.bias_hh.data, val=0)
# Set forget bias to 1
self.leaf_rnn_cell.bias_ih.data.chunk(4)[1].fill_(1)
if self.bidirectional:
init.kaiming_normal(self.leaf_rnn_cell_bw.weight_ih.data)
init.orthogonal(self.leaf_rnn_cell_bw.weight_hh.data)
init.constant(self.leaf_rnn_cell_bw.bias_ih.data, val=0)
init.constant(self.leaf_rnn_cell_bw.bias_hh.data, val=0)
# Set forget bias to 1
self.leaf_rnn_cell_bw.bias_ih.data.chunk(4)[1].fill_(1)
else:
init.kaiming_normal(self.word_linear.weight.data)
init.constant(self.word_linear.bias.data, val=0)
self.treelstm_layer.reset_parameters()
init.normal(self.comp_query.data, mean=0, std=0.01)
def msra_init(net):
'''Init layer parameters.'''
for m in net.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight)
# Modified by lzh @ 201707251408:
# <<< Old:
# if m.bias:
# >>> New:
if m.bias is not None:
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
init.constant(m.weight, 1)
init.constant(m.bias, 0)
elif isinstance(m, nn.Linear):
init.normal(m.weight, std=1e-3)
# Modified by lzh @ 201707241734:
# <<< Old:
# if m.bias:
# >>> New:
if m.bias is not None:
# --- End
init.constant(m.bias, 0)
# Added by lzh @ 201707251404:
def init_params(net):
'''Init layer parameters.'''
for m in net.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight, mode='fan_out')
if m.bias:
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
init.constant(m.weight, 1)
init.constant(m.bias, 0)
elif isinstance(m, nn.Linear):
init.normal(m.weight, std=1e-3)
if m.bias:
init.constant(m.bias, 0)
def linear_params(ni, no):
return cast({'weight': kaiming_normal(torch.Tensor(no, ni)), 'bias': torch.zeros(no)})
def _initialize_weights(self):
vgg16 = torchvision.models.vgg16(pretrained=True)
for m in self.modules():
if isinstance(m, nn.ConvTranspose2d):
assert m.kernel_size[0] == m.kernel_size[1]
m.weight.data = weight_init.kaiming_normal(m.weight.data)
for a, b in zip(vgg16.features, self.features):
if (isinstance(a, nn.Conv2d) and isinstance(b, nn.Conv2d)):
b.weight.data = a.weight.data
b.bias.data = a.bias.data
for i in [0, 3]:
a, b = vgg16.classifier[i], self.classifier[i]
b.weight.data = a.weight.data.view(b.weight.size())
b.bias.data = a.bias.data.view(b.bias.size())
def U_weight_init(ms):
for m in ms.modules():
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
m.weight.data = init.kaiming_normal(m.weight.data, a=0.2)
elif classname.find('ConvTranspose2d') != -1:
m.weight.data = init.kaiming_normal(m.weight.data)
print ('worked!') # TODO: kill this
elif classname.find('BatchNorm') != -1:
m.weight.data.normal_(1.0, 0.02)
m.bias.data.fill_(0)
elif classname.find('Linear') != -1:
m.weight.data = init.kaiming_normal(m.weight.data)
def LR_weight_init(ms):
for m in ms.modules():
classname = m.__class__.__name__
if classname.find('Conv') != -1:
m.weight.data = init.kaiming_normal(m.weight.data, a=0.2)
elif classname.find('BatchNorm') != -1:
m.weight.data.normal_(1.0, 0.02)
m.bias.data.fill_(0)
elif classname.find('Linear') != -1:
m.weight.data = init.kaiming_normal(m.weight.data, a=0.2)
def U_weight_init(ms):
for m in ms.modules():
classname = m.__class__.__name__
if classname.find('Conv2d') != -1:
m.weight.data = init.kaiming_normal(m.weight.data, a=0.2)
elif classname.find('ConvTranspose2d') != -1:
m.weight.data = init.kaiming_normal(m.weight.data)
print ('worked!') # TODO: kill this
elif classname.find('BatchNorm') != -1:
m.weight.data.normal_(1.0, 0.02)
m.bias.data.fill_(0)
elif classname.find('Linear') != -1:
m.weight.data = init.kaiming_normal(m.weight.data)
def LR_weight_init(ms):
for m in ms.modules():
classname = m.__class__.__name__
if classname.find('Conv') != -1:
m.weight.data = init.kaiming_normal(m.weight.data, a=0.2)
elif classname.find('BatchNorm') != -1:
m.weight.data.normal_(1.0, 0.02)
m.bias.data.fill_(0)
elif classname.find('Linear') != -1:
m.weight.data = init.kaiming_normal(m.weight.data, a=0.2)
def __init__(self, block, depth, num_classes):
""" Constructor
Args:
depth: number of layers.
num_classes: number of classes
base_width: base width
"""
super(CifarResNet, self).__init__()
#Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
layer_blocks = (depth - 2) // 6
print ('CifarResNet : Depth : {} , Layers for each block : {}'.format(depth, layer_blocks))
self.num_classes = num_classes
self.conv_1_3x3 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
self.bn_1 = nn.BatchNorm2d(16)
self.inplanes = 16
self.stage_1 = self._make_layer(block, 16, layer_blocks, 1)
self.stage_2 = self._make_layer(block, 32, layer_blocks, 2)
self.stage_3 = self._make_layer(block, 64, layer_blocks, 2)
self.avgpool = nn.AvgPool2d(8)
self.classifier = nn.Linear(64*block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
#m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def __init__(self, block, depth, num_classes):
""" Constructor
Args:
depth: number of layers.
num_classes: number of classes
base_width: base width
"""
super(CifarPreResNet, self).__init__()
#Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
layer_blocks = (depth - 2) // 6
print ('CifarPreResNet : Depth : {} , Layers for each block : {}'.format(depth, layer_blocks))
self.num_classes = num_classes
self.conv_3x3 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
self.inplanes = 16
self.stage_1 = self._make_layer(block, 16, layer_blocks, 1)
self.stage_2 = self._make_layer(block, 32, layer_blocks, 2)
self.stage_3 = self._make_layer(block, 64, layer_blocks, 2)
self.lastact = nn.Sequential(nn.BatchNorm2d(64*block.expansion), nn.ReLU(inplace=True))
self.avgpool = nn.AvgPool2d(8)
self.classifier = nn.Linear(64*block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
#m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def __init__(self, block, depth, cardinality, base_width, num_classes):
super(CifarResNeXt, self).__init__()
#Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
assert (depth - 2) % 9 == 0, 'depth should be one of 29, 38, 47, 56, 101'
layer_blocks = (depth - 2) // 9
self.cardinality = cardinality
self.base_width = base_width
self.num_classes = num_classes
self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False)
self.bn_1 = nn.BatchNorm2d(64)
self.inplanes = 64
self.stage_1 = self._make_layer(block, 64 , layer_blocks, 1)
self.stage_2 = self._make_layer(block, 128, layer_blocks, 2)
self.stage_3 = self._make_layer(block, 256, layer_blocks, 2)
self.avgpool = nn.AvgPool2d(8)
self.classifier = nn.Linear(256*block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def __init__(self, num_classes):
super(CifarCaffeNet, self).__init__()
self.num_classes = num_classes
self.block_1 = nn.Sequential(
nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1),
nn.MaxPool2d(kernel_size=3, stride=2),
nn.ReLU(),
nn.BatchNorm2d(32))
self.block_2 = nn.Sequential(
nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1),
nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),
nn.ReLU(),
nn.AvgPool2d(kernel_size=3, stride=2),
nn.BatchNorm2d(64))
self.block_3 = nn.Sequential(
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
nn.Conv2d(64,128, kernel_size=3, stride=1, padding=1),
nn.ReLU(),
nn.AvgPool2d(kernel_size=3, stride=2),
nn.BatchNorm2d(128))
self.classifier = nn.Linear(128*9, self.num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def __init__(self, block, depth, num_classes):
""" Constructor
Args:
depth: number of layers.
num_classes: number of classes
base_width: base width
"""
super(CifarResNet, self).__init__()
#Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
layer_blocks = (depth - 2) // 6
print ('CifarResNet : Depth : {} , Layers for each block : {}'.format(depth, layer_blocks))
self.num_classes = num_classes
self.conv_1_3x3 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
self.bn_1 = nn.BatchNorm2d(16)
self.inplanes = 16
self.stage_1 = self._make_layer(block, 16, layer_blocks, 1)
self.stage_2 = self._make_layer(block, 32, layer_blocks, 2)
self.stage_3 = self._make_layer(block, 64, layer_blocks, 2)
self.avgpool = nn.AvgPool2d(8)
self.classifier = nn.Linear(64*block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
#m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def conv_init(ni, no, k):
return kaiming_normal(torch.Tensor(no, ni, k, k))
def linear_init(ni, no):
return kaiming_normal(torch.Tensor(no, ni))
def conv_init(ni, no, k):
return kaiming_normal(torch.Tensor(no, ni, k, k))
def linear_init(ni, no):
return kaiming_normal(torch.Tensor(no, ni))
def __init__(self, relu_leakage=0):
super(KaimingNormalWeightsZeroBias, self)\
.__init__(weight_initializer=partial(init.kaiming_normal, a=relu_leakage),
bias_initializer=Constant(0.))
def __init__(self, num_actions):
super().__init__()
self.conv_layers = nn.Sequential(
nn.Conv2d(INPUT_CHANNELS, 32, 8, 4),
nn.ReLU(),
nn.Conv2d(32, 64, 4, 2),
nn.ReLU(),
nn.Conv2d(64, 64, 3, 1),
nn.ReLU()
)
self.fc = nn.Linear(3136, 512)
self.policy_output = nn.Sequential(
nn.Linear(512, num_actions),
nn.Softmax(1)
)
self.value_output = nn.Linear(512, 1)
# init weights and biases
import torch.nn.init as init
for m in self.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def reset_params(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight, mode='fan_out')
if m.bias is not None:
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
init.constant(m.weight, 1)
init.constant(m.bias, 0)
elif isinstance(m, nn.Linear):
init.normal(m.weight, std=0.001)
if m.bias is not None:
init.constant(m.bias, 0)
def reset_params(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight, mode='fan_out')
if m.bias is not None:
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
init.constant(m.weight, 1)
init.constant(m.bias, 0)
elif isinstance(m, nn.Linear):
init.normal(m.weight, std=0.001)
if m.bias is not None:
init.constant(m.bias, 0)
def init_params(net):
'''Init layer parameters.'''
for m in net.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight, mode='fan_out')
if m.bias:
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
init.constant(m.weight, 1)
init.constant(m.bias, 0)
elif isinstance(m, nn.Linear):
init.normal(m.weight, std=1e-3)
if m.bias:
init.constant(m.bias, 0)
def init_params(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
init.kaiming_normal(m.weight, mode='fan_out')
if m.bias is not None:
init.constant(m.bias, 0)
elif isinstance(m, nn.BatchNorm2d):
init.constant(m.weight, 1)
init.constant(m.bias, 0)
elif isinstance(m, nn.Linear):
init.normal(m.weight, std=0.001)
if m.bias is not None:
init.constant(m.bias, 0)
def test_kaiming_normal_errors_on_inputs_smaller_than_2d(self):
for as_variable in [True, False]:
for dims in [0, 1]:
with self.assertRaises(ValueError):
tensor = self._create_random_nd_tensor(dims, size_min=1, size_max=1, as_variable=as_variable)
init.kaiming_normal(tensor)
def test_kaiming_normal(self):
for as_variable in [True, False]:
for use_a in [True, False]:
for dims in [2, 4]:
for mode in ['fan_in', 'fan_out']:
input_tensor = self._create_random_nd_tensor(dims, size_min=20, size_max=25,
as_variable=as_variable)
if use_a:
a = self._random_float(0.1, 2)
init.kaiming_normal(input_tensor, a=a, mode=mode)
else:
a = 0
init.kaiming_normal(input_tensor, mode=mode)
if as_variable:
input_tensor = input_tensor.data
fan_in = input_tensor.size(1)
fan_out = input_tensor.size(0)
if input_tensor.dim() > 2:
fan_in *= input_tensor[0, 0].numel()
fan_out *= input_tensor[0, 0].numel()
if mode == 'fan_in':
n = fan_in
else:
n = fan_out
expected_std = math.sqrt(2.0 / ((1 + a**2) * n))
assert self._is_normal(input_tensor, 0, expected_std)
def he_normal(w, a=0, mode='fan_in'):
return nn.kaiming_normal(w, a=a, mode=mode)
def reset_parameters(self):
if self.use_batchnorm:
self.bn_mlp_input.reset_parameters()
self.bn_mlp_output.reset_parameters()
for i in range(self.num_layers):
linear_layer = self.mlp[i][0]
init.kaiming_normal(linear_layer.weight.data)
init.constant(linear_layer.bias.data, val=0)
init.uniform(self.clf_linear.weight.data, -0.005, 0.005)
init.constant(self.clf_linear.bias.data, val=0)
def reset_parameters(self):
init.kaiming_normal(self.comp_linear.weight.data)
init.constant(self.comp_linear.bias.data, val=0)
