AdamW, concatpooling

bin/train.sh

@@ -4,7 +4,7 @@ export CUDA_VISIBLE_DEVICES=2,3
 RUN_CONFIG=config.yml
 
 
-LOGDIR=/raid/bac/kaggle/logs/recursion_cell/test/rgb_no_crop_512_accum2/se_resnext50_32x4d/
+LOGDIR=/raid/bac/kaggle/logs/recursion_cell/test/c123_s1/se_resnext50_32x4d/
 catalyst-dl run \
     --config=./configs/${RUN_CONFIG} \
     --logdir=$LOGDIR \

src/__init__.py

@@ -5,12 +5,19 @@
 from models import *
 from losses import *
 from callbacks import *
+from optimizers import *
 
 
+# Register models
 registry.Model(cell_resnet)
 registry.Model(cell_senet)
 registry.Model(cell_densenet)
 
+# Register callbacks
 registry.Callback(LabelSmoothCriterionCallback)
 
-registry.Criterion(LabelSmoothingCrossEntropy)
+# Register criterions
+registry.Criterion(LabelSmoothingCrossEntropy)
+
+# Register optimizers
+registry.Optimizer(AdamW)

src/augmentation.py

@@ -1,15 +1,32 @@
 from albumentations import *
 
+import itertools
+
 
 def train_aug(image_size=224):
+    policies = './csv/best_policy.data'
+    with open(policies, 'r') as fid:
+        policies = eval(fid.read())
+        policies = itertools.chain.from_iterable(policies)
+
+    aug_list = []
+    for policy in policies:
+        op_1, params_1 = policy[0]
+        op_2, params_2 = policy[1]
+        aug = Compose([
+            globals().get(op_1)(**params_1),
+            globals().get(op_2)(**params_2),
+        ])
+        aug_list.append(aug)
+
     return Compose([
-        # RandomCrop(448, 448),
-        Resize(image_size, image_size),
         RandomRotate90(),
         Flip(),
         Transpose(),
-        # HorizontalFlip(),
-        # Normalize(),
+        # OneOf(
+        #     aug_list
+        # ),
+        Resize(image_size, image_size)
     ], p=1)
 
 

src/dataset.py

@@ -262,11 +262,11 @@ def __getitem__(self, idx):
 
         image = load_images_as_tensor(channel_paths, dtype=np.float32)
         # image = convert_tensor_to_rgb(image)
-        # image = image / 255
+        image = image / 255
         if self.transform:
             image = self.transform(image=image)['image']
 
-        image = normalize(image, std=std_arr, mean=mean_arr, max_pixel_value=255)
+        # image = normalize(image, std=std_arr, mean=mean_arr, max_pixel_value=255)
         image = np.transpose(image, (2, 0, 1)).astype(np.float32)
 
         if self.mode == 'train':

src/make_submission.py

@@ -35,7 +35,7 @@ def predict(model, loader):
 def predict_all():
     test_csv = '/raid/data/kaggle/recursion-cellular-image-classification/test.csv'
     # test_csv = './csv/valid_0.csv'
-    log_dir = "/raid/bac/kaggle/logs/recursion_cell/test/rgb_no_crop_512/se_resnext50_32x4d/"
+    log_dir = "/raid/bac/kaggle/logs/recursion_cell/test/c123_s1_concatpool/se_resnext50_32x4d/"
     root = "/raid/data/kaggle/recursion-cellular-image-classification/"
     site = 1
     channels = [1,2,3]
@@ -75,8 +75,8 @@ def predict_all():
     submission = df.copy()
     submission['sirna'] = all_preds.astype(int)
     os.makedirs("submission", exist_ok=True)
-    submission.to_csv('./submission/se_resnext50_32x4d_no_crop_512_test.csv', index=False, columns=['id_code', 'sirna'])
-    np.save("./submission/se_resnext50_32x4d_no_crop_512_test.npy", pred)
+    submission.to_csv('./submission/se_resnext50_32x4d_c123_s1_concatpool.csv', index=False, columns=['id_code', 'sirna'])
+    np.save("./submission/se_resnext50_32x4d_c123_s1_concatpool.npy", pred)
 
 
 if __name__ == '__main__':

src/models/senet.py

@@ -1,14 +1,17 @@
 import torch.nn as nn
 import pretrainedmodels
 from cnn_finetune import make_model
+from .utils import *
 
 
 def cell_senet(model_name='se_resnext50', num_classes=1108, n_channels=6):
     # model = pretrainedmodels.__dict__[model_name](num_classes=1000, pretrained='imagenet')
     model = make_model(
         model_name=model_name,
         num_classes=num_classes,
-        pretrained=True
+        pretrained=True,
+        # pool=GlobalConcatPool2d(),
+        # classifier_factory=make_classifier
     )
     # print(model)
     conv1 = model._features[0].conv1

src/models/utils.py

@@ -0,0 +1,16 @@
+from catalyst.contrib.modules.pooling import GlobalConcatPool2d
+from catalyst.contrib.modules.common import Flatten
+import torch.nn as nn
+
+
+def make_classifier(in_features, num_classes):
+    return nn.Sequential(
+        Flatten(),
+        nn.BatchNorm1d(in_features * 2),
+        nn.Dropout(0.5),
+        nn.Linear(in_features * 2, 1024),
+        nn.ReLU(inplace=True),
+        nn.BatchNorm1d(1024),
+        nn.Dropout(0.5),
+        nn.Linear(1024, num_classes),
+    )

src/optimizers.py

@@ -0,0 +1,114 @@
+import math
+import torch
+from torch.optim.optimizer import Optimizer
+
+
+class AdamW(Optimizer):
+    r"""Implements AdamW algorithm.
+
+    The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_.
+    The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_.
+
+    Arguments:
+        params (iterable): iterable of parameters to optimize or dicts defining
+            parameter groups
+        lr (float, optional): learning rate (default: 1e-3)
+        betas (Tuple[float, float], optional): coefficients used for computing
+            running averages of gradient and its square (default: (0.9, 0.999))
+        eps (float, optional): term added to the denominator to improve
+            numerical stability (default: 1e-8)
+        weight_decay (float, optional): weight decay coefficient (default: 1e-2)
+        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
+            algorithm from the paper `On the Convergence of Adam and Beyond`_
+            (default: False)
+
+    .. _Adam\: A Method for Stochastic Optimization:
+        https://arxiv.org/abs/1412.6980
+    .. _Decoupled Weight Decay Regularization:
+        https://arxiv.org/abs/1711.05101
+    .. _On the Convergence of Adam and Beyond:
+        https://openreview.net/forum?id=ryQu7f-RZ
+    """
+
+    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
+                 weight_decay=1e-2, amsgrad=False):
+        if not 0.0 <= lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 <= eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay, amsgrad=amsgrad)
+        super(AdamW, self).__init__(params, defaults)
+
+    def __setstate__(self, state):
+        super(AdamW, self).__setstate__(state)
+        for group in self.param_groups:
+            group.setdefault('amsgrad', False)
+
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Arguments:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            loss = closure()
+
+        for group in self.param_groups:
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+
+                # Perform stepweight decay
+                p.data.mul_(1 - group['lr'] * group['weight_decay'])
+
+                # Perform optimization step
+                grad = p.grad.data
+                if grad.is_sparse:
+                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
+                amsgrad = group['amsgrad']
+
+                state = self.state[p]
+
+                # State initialization
+                if len(state) == 0:
+                    state['step'] = 0
+                    # Exponential moving average of gradient values
+                    state['exp_avg'] = torch.zeros_like(p.data)
+                    # Exponential moving average of squared gradient values
+                    state['exp_avg_sq'] = torch.zeros_like(p.data)
+                    if amsgrad:
+                        # Maintains max of all exp. moving avg. of sq. grad. values
+                        state['max_exp_avg_sq'] = torch.zeros_like(p.data)
+
+                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
+                if amsgrad:
+                    max_exp_avg_sq = state['max_exp_avg_sq']
+                beta1, beta2 = group['betas']
+
+                state['step'] += 1
+
+                # Decay the first and second moment running average coefficient
+                exp_avg.mul_(beta1).add_(1 - beta1, grad)
+                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
+                if amsgrad:
+                    # Maintains the maximum of all 2nd moment running avg. till now
+                    torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
+                    # Use the max. for normalizing running avg. of gradient
+                    denom = max_exp_avg_sq.sqrt().add_(group['eps'])
+                else:
+                    denom = exp_avg_sq.sqrt().add_(group['eps'])
+
+                bias_correction1 = 1 - beta1 ** state['step']
+                bias_correction2 = 1 - beta2 ** state['step']
+                step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1
+
+                p.data.addcdiv_(-step_size, exp_avg, denom)
+
+        return loss