release code about MNIST

compare.py

@@ -0,0 +1,145 @@
+import matplotlib.pyplot as plt
+import os
+import sys
+import numpy as np
+from collections import OrderedDict
+
+def savefig(fname, dpi=None):
+    dpi = 150 if dpi == None else dpi
+    plt.savefig(fname, dpi=dpi)
+
+
+class Logger(object):
+    '''Save training process to log file with simple plot function.'''
+    def __init__(self, fpath, title=None, resume=False): 
+        self.file = None
+        self.resume = resume
+        self.title = '' if title == None else title
+        if fpath is not None:
+            if resume: 
+                self.file = open(fpath, 'r') 
+                name = self.file.readline()
+                self.names = name.rstrip().split('\t')
+                self.numbers = {}
+                for _, name in enumerate(self.names):
+                    self.numbers[name] = []
+
+                for numbers in self.file:
+                    numbers = numbers.rstrip().split('\t')
+                    for i in range(0, len(numbers)):
+                        self.numbers[self.names[i]].append(numbers[i])
+                self.file.close()
+                self.file = open(fpath, 'a')  
+            else:
+                self.file = open(fpath, 'w')
+
+    def set_names(self, names):
+        if self.resume: 
+            pass
+        # initialize numbers as empty list
+        self.numbers = {}
+        self.names = names
+        for _, name in enumerate(self.names):
+            self.file.write(name)
+            self.file.write('\t')
+            self.numbers[name] = []
+        self.file.write('\n')
+        self.file.flush()
+
+
+    def append(self, numbers):
+        assert len(self.names) == len(numbers), 'Numbers do not match names'
+        for index, num in enumerate(numbers):
+            self.file.write("{0:.6f}".format(num))
+            self.file.write('\t')
+            self.numbers[self.names[index]].append(num)
+        self.file.write('\n')
+        self.file.flush()
+
+    def plot(self, names=None):   
+        names = self.names if names == None else names
+        numbers = self.numbers
+        for _, name in enumerate(names):
+            x = np.arange(len(numbers[name]))
+            plt.plot(x, np.asarray(numbers[name]))         
+        plt.legend([self.title + '(' + name + ')' for name in names])
+        plt.grid(True)
+
+    def close(self):
+        if self.file is not None:
+            self.file.close()
+
+def plot_overlap(logger, names, axs, indexes):
+    names = logger.names if names == None else names
+    numbers = logger.numbers
+    for _, name in enumerate(names):
+        x = np.arange(len(numbers[name]))
+        axs[indexes[0],indexes[1]].plot(x, np.asarray(numbers[name]))
+
+    return [logger.title for name in names]       
+
+class LoggerMonitor(object):
+    '''Load and visualize multiple logs.'''
+    def __init__ (self, paths):
+        '''paths is a distionary with {name:filepath} pair'''
+        self.loggers = []
+        for title, path in paths.items():
+            logger = Logger(path, title=title, resume=True)
+            self.loggers.append(logger)
+
+    def plot(self, names, axs, indexes):
+
+        legend_text = []
+        for logger in self.loggers:
+            legend_text += plot_overlap(logger, names, axs, indexes)
+
+        axs[indexes[0],indexes[1]].set_title(names[0])
+        return legend_text
+
+paths = OrderedDict()
+fields = [['Train Loss'],['Valid Loss'], ['Train Acc.'],['Valid Acc.']]  
+
+paths['SGD-Momentum']='momentum.txt'
+I=float(1)
+learning_rate = 0.01
+
+name = 'pid.txt'
+paths['PID']=name 
+fig, axs = plt.subplots(2, 2)   
+
+i = 0
+indexes=[0,0]
+
+field = fields[i]
+monitor = LoggerMonitor(paths)
+legend_text = monitor.plot(names=field, axs=axs, indexes=indexes)
+axs[indexes[0],indexes[1]].legend(legend_text, bbox_to_anchor=(1., 1.0), loc=1, borderaxespad=0.)
+axs[indexes[0],indexes[1]].set_xlabel('Epoch')
+
+i = 1
+indexes=[0,1]
+field = fields[i]
+monitor = LoggerMonitor(paths)
+legend_text = monitor.plot(names=field, axs=axs, indexes=indexes)
+axs[indexes[0],indexes[1]].legend(legend_text, bbox_to_anchor=(1., 1.0), loc=1, borderaxespad=0.)
+axs[indexes[0],indexes[1]].set_xlabel('Epoch')
+
+i = 2
+indexes=[1,0]
+field = fields[i]
+monitor = LoggerMonitor(paths)
+legend_text = monitor.plot(names=field, axs=axs, indexes=indexes)
+axs[indexes[0],indexes[1]].legend(legend_text, bbox_to_anchor=(1., 0.), loc=4, borderaxespad=0.)
+axs[indexes[0],indexes[1]].set_xlabel('Epoch')
+
+i = 3
+indexes=[1,1]
+field = fields[i]
+monitor = LoggerMonitor(paths)
+legend_text = monitor.plot(names=field, axs=axs, indexes=indexes)
+axs[indexes[0],indexes[1]].legend(legend_text, bbox_to_anchor=(1., 0.), loc=4, borderaxespad=0.)
+axs[indexes[0],indexes[1]].set_xlabel('Epoch')
+
+fig.tight_layout()
+
+savefig('moment_vs_pid.pdf')

mnist_moment.py

@@ -0,0 +1,106 @@
+import torch
+import torch.nn as nn
+import torchvision.datasets as dsets
+import torchvision.transforms as transforms
+from torch.autograd import Variable
+from torch.optim.sgd import SGD
+import os
+from utils import Bar, Logger, AverageMeter, accuracy, mkdir_p, savefig
+import torch.nn.functional as F
+# Hyper Parameters 
+input_size = 784
+hidden_size = 1000
+num_classes = 10
+num_epochs = 20
+batch_size = 100
+learning_rate = 0.01
+
+logger = Logger('momentum.txt', title='mnist')
+logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
+
+# MNIST Dataset 
+train_dataset = dsets.MNIST(root='./data', 
+                            train=True, 
+                            transform=transforms.ToTensor(),  
+                            download=True)
+
+test_dataset = dsets.MNIST(root='./data', 
+                           train=False, 
+                           transform=transforms.ToTensor())
+
+# Data Loader (Input Pipeline)
+train_loader = torch.utils.data.DataLoader(dataset=train_dataset, 
+                                           batch_size=batch_size, 
+                                           shuffle=True)
+
+test_loader = torch.utils.data.DataLoader(dataset=test_dataset, 
+                                          batch_size=batch_size, 
+                                          shuffle=False)
+
+# Neural Network Model (1 hidden layer)
+class Net(nn.Module):
+    def __init__(self, input_size, hidden_size, num_classes):
+        super(Net, self).__init__()
+        self.fc1 = nn.Linear(input_size, hidden_size) 
+        self.fc2 = nn.Linear(hidden_size, num_classes)  
+
+    def forward(self, x):
+        out = self.fc1(x)
+        out = F.relu(out)
+        out = self.fc2(out)
+        return out
+
+net = Net(input_size, hidden_size, num_classes)
+net.cuda()   
+net.train()    
+# Loss and Optimizer
+criterion = nn.CrossEntropyLoss()  
+#optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)  
+optimizer = SGD(net.parameters(), lr=learning_rate, weight_decay=0.0001, momentum=0.9)
+# Train the Model
+for epoch in range(num_epochs):
+
+    train_loss_log = AverageMeter()
+    train_acc_log = AverageMeter()
+    val_loss_log = AverageMeter()
+    val_acc_log = AverageMeter()    
+    for i, (images, labels) in enumerate(train_loader):  
+        # Convert torch tensor to Variable
+        images = Variable(images.view(-1, 28*28).cuda())
+        labels = Variable(labels.cuda())
+
+        # Forward + Backward + Optimize
+        optimizer.zero_grad()  # zero the gradient buffer
+        outputs = net(images)
+        train_loss = criterion(outputs, labels)
+        train_loss.backward()
+        optimizer.step()
+        prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5))
+        train_loss_log.update(train_loss.data[0], images.size(0))
+        train_acc_log.update(prec1[0], images.size(0))
+
+        if (i+1) % 100 == 0:
+            print ('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Acc: %.8f' 
+                   %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, train_loss_log.avg, train_acc_log.avg))
+
+    # Test the Model
+    net.eval()
+    correct = 0
+    loss = 0
+    total = 0
+    for images, labels in test_loader:
+        images = Variable(images.view(-1, 28*28)).cuda()
+        labels = Variable(labels).cuda()
+        outputs = net(images)
+        test_loss = criterion(outputs, labels)
+        val_loss_log.update(test_loss.data[0], images.size(0))
+        prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5))
+        val_acc_log.update(prec1[0], images.size(0))
+
+    logger.append([learning_rate, train_loss_log.avg, val_loss_log.avg, train_acc_log.avg, val_acc_log.avg])
+    print('Accuracy of the network on the 10000 test images: %.8f %%' % (val_acc_log.avg))
+    print('Loss of the network on the 10000 test images: %.8f' % (val_loss_log.avg))
+
+logger.close()
+logger.plot()
+

mnist_pid.py

@@ -0,0 +1,112 @@
+import torch
+import torch.nn as nn
+import torchvision.datasets as dsets
+import torchvision.transforms as transforms
+from torch.autograd import Variable
+from pid import PIDOptimizer
+import os
+import numpy as np
+from utils import Bar, Logger, AverageMeter, accuracy, mkdir_p, savefig
+import torch.nn.functional as F
+# Hyper Parameters 
+input_size = 784
+hidden_size = 1000
+num_classes = 10
+num_epochs = 20
+batch_size = 100
+learning_rate = 0.01
+
+I=1
+I = float(I)
+D = 200
+D = float(D)
+
+
+logger = Logger('pid.txt', title='mnist')
+logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
+
+# MNIST Dataset 
+train_dataset = dsets.MNIST(root='./data', 
+                            train=True, 
+                            transform=transforms.ToTensor(),  
+                            download=True)
+
+test_dataset = dsets.MNIST(root='./data', 
+                           train=False, 
+                           transform=transforms.ToTensor())
+
+# Data Loader (Input Pipeline)
+train_loader = torch.utils.data.DataLoader(dataset=train_dataset, 
+                                           batch_size=batch_size, 
+                                           shuffle=True)
+
+test_loader = torch.utils.data.DataLoader(dataset=test_dataset, 
+                                          batch_size=batch_size, 
+                                          shuffle=False)
+
+# Neural Network Model (1 hidden layer)
+class Net(nn.Module):
+    def __init__(self, input_size, hidden_size, num_classes):
+        super(Net, self).__init__()
+        self.fc1 = nn.Linear(input_size, hidden_size) 
+        self.fc2 = nn.Linear(hidden_size, num_classes)  
+
+    def forward(self, x):
+        out = self.fc1(x)
+        out = F.relu(out)
+        out = self.fc2(out)
+        return out
+
+net = Net(input_size, hidden_size, num_classes)
+net.cuda()   
+net.train()    
+# Loss and Optimizer
+criterion = nn.CrossEntropyLoss()  
+optimizer = PIDOptimizer(net.parameters(), lr=learning_rate, weight_decay=0.0001, momentum=0.9, I=I, D=D)
+# Train the Model
+for epoch in range(num_epochs):
+
+    train_loss_log = AverageMeter()
+    train_acc_log = AverageMeter()
+    val_loss_log = AverageMeter()
+    val_acc_log = AverageMeter()    
+    for i, (images, labels) in enumerate(train_loader):  
+        # Convert torch tensor to Variable
+        images = Variable(images.view(-1, 28*28).cuda())
+        labels = Variable(labels.cuda())
+
+        # Forward + Backward + Optimize
+        optimizer.zero_grad()  # zero the gradient buffer
+        outputs = net(images)
+        train_loss = criterion(outputs, labels)
+        train_loss.backward()
+        optimizer.step()
+        prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5))
+        train_loss_log.update(train_loss.data[0], images.size(0))
+        train_acc_log.update(prec1[0], images.size(0))
+
+        if (i+1) % 100 == 0:
+            print ('Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Acc: %.8f' 
+                   %(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, train_loss_log.avg, train_acc_log.avg))
+
+    # Test the Model
+    net.eval()
+    correct = 0
+    loss = 0
+    total = 0
+    for images, labels in test_loader:
+        images = Variable(images.view(-1, 28*28)).cuda()
+        labels = Variable(labels).cuda()
+        outputs = net(images)
+        test_loss = criterion(outputs, labels)
+        val_loss_log.update(test_loss.data[0], images.size(0))
+        prec1, prec5 = accuracy(outputs.data, labels.data, topk=(1, 5))
+        val_acc_log.update(prec1[0], images.size(0))
+
+    logger.append([learning_rate, train_loss_log.avg, val_loss_log.avg, train_acc_log.avg, val_acc_log.avg])
+    print('Accuracy of the network on the 10000 test images: %d %%' % (val_acc_log.avg))
+    print('Loss of the network on the 10000 test images: %.8f' % (val_loss_log.avg))
+
+logger.close()
+logger.plot()
+

momentum.txt

@@ -0,0 +1,21 @@
+Learning Rate	Train Loss	Valid Loss	Train Acc.	Valid Acc.	
+0.010000	0.516547	0.289208	86.581667	91.600000	
+0.010000	0.259415	0.216152	92.545000	93.950000	
+0.010000	0.201986	0.177764	94.343333	94.780000	
+0.010000	0.165051	0.149667	95.318333	95.600000	
+0.010000	0.139030	0.128418	96.070000	96.220000	
+0.010000	0.119939	0.116352	96.710000	96.600000	
+0.010000	0.104811	0.104972	97.101667	96.880000	
+0.010000	0.093801	0.096976	97.466667	97.120000	
+0.010000	0.084180	0.094102	97.731667	97.140000	
+0.010000	0.076191	0.085866	97.956667	97.510000	
+0.010000	0.069832	0.081906	98.118333	97.590000	
+0.010000	0.064328	0.079128	98.340000	97.710000	
+0.010000	0.059059	0.077403	98.468333	97.740000	
+0.010000	0.055209	0.074700	98.570000	97.780000	
+0.010000	0.051301	0.072494	98.680000	97.880000	
+0.010000	0.047872	0.071196	98.815000	97.770000	
+0.010000	0.044999	0.071158	98.900000	97.790000	
+0.010000	0.042177	0.066217	98.998333	97.990000	
+0.010000	0.039857	0.066495	99.031667	97.970000	
+0.010000	0.037694	0.065470	99.106667	97.950000