def get_loader(config):
"""Builds and returns Dataloader for MNIST and SVHN dataset."""
transform = transforms.Compose([
transforms.Scale(config.image_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
svhn = datasets.SVHN(root=config.svhn_path, download=True, transform=transform)
mnist = datasets.MNIST(root=config.mnist_path, download=True, transform=transform)
svhn_loader = torch.utils.data.DataLoader(dataset=svhn,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
mnist_loader = torch.utils.data.DataLoader(dataset=mnist,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
return svhn_loader, mnist_loader
python类MNIST的实例源码
def get_mnist(train):
"""Get MNIST dataset loader."""
# image pre-processing
pre_process = transforms.Compose([transforms.ToTensor(),
transforms.Normalize(
mean=params.dataset_mean,
std=params.dataset_std)])
# dataset and data loader
mnist_dataset = datasets.MNIST(root=params.data_root,
train=train,
transform=pre_process,
download=True)
mnist_data_loader = torch.utils.data.DataLoader(
dataset=mnist_dataset,
batch_size=params.batch_size,
shuffle=True)
return mnist_data_loader
def mnist(self):
norm_mean = [0.1307]
norm_std = [0.3081]
train_loader = torch.utils.data.DataLoader(
dsets.MNIST("/home/dataset/mnist", train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std)
])),
batch_size=self.train_batch_size, shuffle=True,
num_workers=self.n_threads,
pin_memory=False
)
test_loader = torch.utils.data.DataLoader(
dsets.MNIST("/home/dataset/mnist", train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std)
])),
batch_size=self.test_batch_size, shuffle=True,
num_workers=self.n_threads,
pin_memory=False
)
return train_loader, test_loader
def train(rank, args, model):
torch.manual_seed(args.seed + rank)
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batch_size, shuffle=True, num_workers=1)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batch_size, shuffle=True, num_workers=1)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
for epoch in range(1, args.epochs + 1):
train_epoch(epoch, args, model, train_loader, optimizer)
test_epoch(model, test_loader)
def __init__(self, batch_size):
self.batch_size = batch_size
train_dataset = dsets.MNIST(root="./data",
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = dsets.MNIST(root="./data",
train=False,
transform=transforms.ToTensor())
self.train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
self.test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=True)
def get_loaders(loader_batchsize, **kwargs):
arguments=kwargs['arguments']
data = arguments.data
if data == 'mnist':
kwargs = {'num_workers': 1, 'pin_memory': True} if arguments.cuda else {}
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
#transforms.Normalize((0,), (1,))
])),
batch_size=loader_batchsize, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=False, transform=transforms.Compose([
transforms.ToTensor(),
#transforms.Normalize((7,), (0.3081,))
])),
batch_size=loader_batchsize, shuffle=True, **kwargs)
return train_loader, test_loader
def train(rank, args, model):
torch.manual_seed(args.seed + rank)
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batch_size, shuffle=True, num_workers=1)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batch_size, shuffle=True, num_workers=1)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
for epoch in range(1, args.epochs + 1):
train_epoch(epoch, args, model, train_loader, optimizer)
test_epoch(model, test_loader)
def train(epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
if use_cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % 500 == 0:
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
100. * batch_idx / len(train_loader), loss.data[0]))
#
# A simple test procedure to measure STN the performances on MNIST.
#
def get_dataset(name, split='train', transform=None,
target_transform=None, download=True, datasets_path=__DATASETS_DEFAULT_PATH):
train = (split == 'train')
root = os.path.join(datasets_path, name)
if name == 'cifar10':
return datasets.CIFAR10(root=root,
train=train,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'cifar100':
return datasets.CIFAR100(root=root,
train=train,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'mnist':
return datasets.MNIST(root=root,
train=train,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'stl10':
return datasets.STL10(root=root,
split=split,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'imagenet':
if train:
root = os.path.join(root, 'train')
else:
root = os.path.join(root, 'val')
return datasets.ImageFolder(root=root,
transform=transform,
target_transform=target_transform)
def get_loader(config):
"""Builds and returns Dataloader for MNIST and SVHN dataset."""
transform = transforms.Compose([
transforms.Scale(config.image_size),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
svhn = datasets.SVHN(root=config.svhn_path, download=True, transform=transform, split='train')
mnist = datasets.MNIST(root=config.mnist_path, download=True, transform=transform, train=True)
svhn_test = datasets.SVHN(root=config.svhn_path, download=True, transform=transform, split='test')
mnist_test = datasets.MNIST(root=config.mnist_path, download=True, transform=transform, train=False)
svhn_loader = torch.utils.data.DataLoader(dataset=svhn,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
mnist_loader = torch.utils.data.DataLoader(dataset=mnist,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
svhn_test_loader = torch.utils.data.DataLoader(dataset=svhn_test,
batch_size=config.batch_size,
shuffle=False,
num_workers=config.num_workers)
mnist_test_loader = torch.utils.data.DataLoader(dataset=mnist_test,
batch_size=config.batch_size,
shuffle=False,
num_workers=config.num_workers)
return svhn_loader, mnist_loader, svhn_test_loader, mnist_test_loader
def __init__(self, dataset, train_batch_size, test_batch_size, n_threads=4, ten_crop=False):
self.dataset = dataset
self.train_batch_size = train_batch_size
self.test_batch_size = test_batch_size
self.n_threads = n_threads
self.ten_crop = ten_crop
if self.dataset == "cifar10" or self.dataset == "cifar100":
print "|===>Creating Cifar Data Loader"
self.train_loader, self.test_loader = self.cifar(dataset=self.dataset)
elif self.dataset == "mnist":
print "|===>Creating MNIST Data Loader"
self.train_loader, self.test_loader = self.mnist()
else:
assert False, "invalid data set"
def get_mnist(train):
"""Get MNIST dataset loader."""
# image pre-processing
pre_process = transforms.Compose([transforms.ToTensor(),
transforms.Normalize(
mean=params.dataset_mean,
std=params.dataset_std)])
# dataset and data loader
mnist_dataset = datasets.MNIST(root=params.data_root,
train=train,
transform=pre_process,
download=True)
mnist_data_loader = torch.utils.data.DataLoader(
dataset=mnist_dataset,
batch_size=params.batch_size,
shuffle=True)
return mnist_data_loader
def get_dataset(name, split='train', transform=None,
target_transform=None, download=True, datasets_path=__DATASETS_DEFAULT_PATH):
train = (split == 'train')
root = os.path.join(datasets_path, name)
if name == 'cifar10':
return datasets.CIFAR10(root=root,
train=train,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'cifar100':
return datasets.CIFAR100(root=root,
train=train,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'mnist':
return datasets.MNIST(root=root,
train=train,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'stl10':
return datasets.STL10(root=root,
split=split,
transform=transform,
target_transform=target_transform,
download=download)
elif name == 'imagenet':
if train:
root = os.path.join(root, 'train')
else:
root = os.path.join(root, 'val')
return datasets.ImageFolder(root=root,
transform=transform,
target_transform=target_transform)
MNIST_with_centerloss.py 文件源码
项目：MNIST_center_loss_pytorch
作者: jxgu1016
项目源码
文件源码
阅读 22
收藏 0
点赞 0
评论 0
def main():
if torch.cuda.is_available():
use_cuda = True
else: use_cuda = False
# Dataset
trainset = datasets.MNIST('../../data', download=True,train=True, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))]))
train_loader = DataLoader(trainset, batch_size=128, shuffle=True, num_workers=4)
# Model
model = Net()
# NLLLoss
nllloss = nn.NLLLoss() #CrossEntropyLoss = log_softmax + NLLLoss
# CenterLoss
loss_weight = 1.0
centerloss = CenterLoss(10,2,loss_weight)
if use_cuda:
nllloss = nllloss.cuda()
centerloss = centerloss.cuda()
model = model.cuda()
criterion = [nllloss, centerloss]
# optimzer4nn
optimizer4nn = optim.SGD(model.parameters(),lr=0.001,momentum=0.9, weight_decay=0.0005)
sheduler = lr_scheduler.StepLR(optimizer4nn,20,gamma=0.8)
# optimzer4center
optimzer4center = optim.SGD(centerloss.parameters(), lr =0.5)
for epoch in range(50):
sheduler.step()
# print optimizer4nn.param_groups[0]['lr']
train(train_loader, model, criterion, [optimizer4nn, optimzer4center], epoch+1, use_cuda)
def MNIST_loader(root, image_size, normalize=True):
"""
Function to load torchvision dataset object based on just image size
Args:
root = If your dataset is downloaded and ready to use, mention the location of this folder. Else, the dataset will be downloaded to this location
image_size = Size of every image
normalize = Requirement to normalize the image. Default is true
"""
transformations = [transforms.Scale(image_size), transforms.ToTensor()]
if normalize == True:
transformations.append(transforms.Normalize((0.5, ), (0.5, )))
mnist_data = dset.MNIST(root=root, download=True, transform=transforms.Compose(transformations))
return mnist_data
def get(batch_size, data_root='/tmp/public_dataset/pytorch', train=True, val=True, **kwargs):
data_root = os.path.expanduser(os.path.join(data_root, 'mnist-data'))
kwargs.pop('input_size', None)
num_workers = kwargs.setdefault('num_workers', 1)
print("Building MNIST data loader with {} workers".format(num_workers))
ds = []
if train:
train_loader = torch.utils.data.DataLoader(
datasets.MNIST(root=data_root, train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=batch_size, shuffle=True, **kwargs)
ds.append(train_loader)
if val:
test_loader = torch.utils.data.DataLoader(
datasets.MNIST(root=data_root, train=False, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=batch_size, shuffle=True, **kwargs)
ds.append(test_loader)
ds = ds[0] if len(ds) == 1 else ds
return ds
def execute(self):
batch_size = 50
train_loader = torch.utils.data.DataLoader(datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor()), batch_size=batch_size, shuffle=True)
self.data = train_loader
for port in self.outPorts:
port.transferData(self.data)
def __init__(self, batchsize, train=True):
Dataset.__init__(self)
data_root = join(dirname(realpath(__file__)), 'MNIST_data')
self.name = "mnist"
self.range = [0.0, 1.0]
self.data_dims = [1, 28, 28]
self.batchsize = batchsize
self.data = dsets.MNIST(root=data_root,
download=True,
train=train,
transform=transforms.Compose([
transforms.ToTensor()]))
self.dataloder = tdata.DataLoader(self.data, self.batchsize, shuffle=True)
self.iter = iter(self.dataloder)
self._index = 0
def __init__(self, mode, sup_num, use_cuda=True, *args, **kwargs):
super(MNISTCached, self).__init__(train=mode in ["sup", "unsup", "valid"], *args, **kwargs)
# transformations on MNIST data (normalization and one-hot conversion for labels)
def transform(x):
return fn_x_mnist(x, use_cuda)
def target_transform(y):
return fn_y_mnist(y, use_cuda)
self.mode = mode
assert mode in ["sup", "unsup", "test", "valid"], "invalid train/test option values"
if mode in ["sup", "unsup", "valid"]:
# transform the training data if transformations are provided
if transform is not None:
self.train_data = (transform(self.train_data.float()))
if target_transform is not None:
self.train_labels = (target_transform(self.train_labels))
if MNISTCached.train_data_sup is None:
if sup_num is None:
assert mode == "unsup"
MNISTCached.train_data_unsup, MNISTCached.train_labels_unsup = \
self.train_data, self.train_labels
else:
MNISTCached.train_data_sup, MNISTCached.train_labels_sup, \
MNISTCached.train_data_unsup, MNISTCached.train_labels_unsup, \
MNISTCached.data_valid, MNISTCached.labels_valid = \
split_sup_unsup_valid(self.train_data, self.train_labels, sup_num)
if mode == "sup":
self.train_data, self.train_labels = MNISTCached.train_data_sup, MNISTCached.train_labels_sup
elif mode == "unsup":
self.train_data = MNISTCached.train_data_unsup
# making sure that the unsupervised labels are not available to inference
self.train_labels = (torch.Tensor(
MNISTCached.train_labels_unsup.shape[0]).view(-1, 1)) * np.nan
else:
self.train_data, self.train_labels = MNISTCached.data_valid, MNISTCached.labels_valid
else:
# transform the testing data if transformations are provided
if transform is not None:
self.test_data = (transform(self.test_data.float()))
if target_transform is not None:
self.test_labels = (target_transform(self.test_labels))
def get_data(args, train_flag=True):
transform = transforms.Compose([
transforms.Scale(args.image_size),
transforms.CenterCrop(args.image_size),
transforms.ToTensor(),
transforms.Normalize(
(0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
if args.dataset in ['imagenet', 'folder', 'lfw']:
dataset = dset.ImageFolder(root=args.dataroot,
transform=transform)
elif args.dataset == 'lsun':
dataset = dset.LSUN(db_path=args.dataroot,
classes=['bedroom_train'],
transform=transform)
elif args.dataset == 'cifar10':
dataset = dset.CIFAR10(root=args.dataroot,
download=True,
train=train_flag,
transform=transform)
elif args.dataset == 'cifar100':
dataset = dset.CIFAR100(root=args.dataroot,
download=True,
train=train_flag,
transform=transform)
elif args.dataset == 'mnist':
dataset = dset.MNIST(root=args.dataroot,
download=True,
train=train_flag,
transform=transform)
elif args.dataset == 'celeba':
imdir = 'train' if train_flag else 'val'
dataroot = os.path.join(args.dataroot, imdir)
if args.image_size != 64:
raise ValueError('the image size for CelebA dataset need to be 64!')
dataset = FolderWithImages(root=dataroot,
input_transform=transforms.Compose([
ALICropAndScale(),
transforms.ToTensor(),
transforms.Normalize(
(0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
]),
target_transform=transforms.ToTensor()
)
else:
raise ValueError("Unknown dataset %s" % (args.dataset))
return dataset
def get_loaders(args):
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
if args.dataset == 'mnist':
trainLoader = torch.utils.data.DataLoader(
dset.MNIST('data/mnist', train=True, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batchSz, shuffle=True, **kwargs)
testLoader = torch.utils.data.DataLoader(
dset.MNIST('data/mnist', train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=args.batchSz, shuffle=False, **kwargs)
elif args.dataset == 'cifar-10':
normMean = [0.49139968, 0.48215827, 0.44653124]
normStd = [0.24703233, 0.24348505, 0.26158768]
normTransform = transforms.Normalize(normMean, normStd)
trainTransform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normTransform
])
testTransform = transforms.Compose([
transforms.ToTensor(),
normTransform
])
trainLoader = DataLoader(
dset.CIFAR10(root='data/cifar', train=True, download=True,
transform=trainTransform),
batch_size=args.batchSz, shuffle=True, **kwargs)
testLoader = DataLoader(
dset.CIFAR10(root='data/cifar', train=False, download=True,
transform=testTransform),
batch_size=args.batchSz, shuffle=False, **kwargs)
else:
assert(False)
return trainLoader, testLoader
def execute(self):
model = MnistModel()
batch_size = 50
train_loader = torch.utils.data.DataLoader(datasets.MNIST('data', train=True, download=True, transform=transforms.ToTensor()), batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(datasets.MNIST('data', train=False, transform=transforms.ToTensor()), batch_size=1000)
for p in model.parameters():
print(p.size())
optimizer = optim.Adam(model.parameters(), lr=0.0001)
model.train()
train_loss = []
train_accu = []
i = 0
for epoch in range(15):
for data, target in train_loader:
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward() # calc gradients
train_loss.append(loss.data[0])
optimizer.step() # update gradients
prediction = output.data.max(1)[1] # first column has actual prob.
accuracy = prediction.eq(target.data).sum()/batch_size*100
train_accu.append(accuracy)
if i % 1000 == 0:
print('Train Step: {}\tLoss: {:.3f}\tAccuracy: {:.3f}'.format(i, loss.data[0], accuracy))
i += 1
plt.plot(np.arange(len(train_loss)), train_loss)
plt.plot(np.arange(len(train_accu)), train_accu)
model.eval()
correct = 0
for data, target in test_loader:
data, target = Variable(data, volatile=True), Variable(target)
output = model(data)
prediction = output.data.max(1)[1]
correct += prediction.eq(target.data).sum()
print('\nTest set: Accuracy: {:.2f}%'.format(100. * correct / len(test_loader.dataset)))
self.data = np.random.rand(self.dim[0],self.dim[1])
for port in self.outPorts:
port.transferData(self.data)
