Affine augmentation. warmup steps = 2 epochs. That increases 3% LB

bin/train.sh

@@ -4,7 +4,7 @@ export CUDA_VISIBLE_DEVICES=2,3
 RUN_CONFIG=config.yml
 
 
-LOGDIR=/raid/bac/kaggle/logs/recursion_cell/test/c1234_s1_smooth_nadam_rndsite_64/se_resnext50_32x4d/
+LOGDIR=/raid/bac/kaggle/logs/recursion_cell/test/c1234_s1_affine_warmup/se_resnext50_32x4d/
 catalyst-dl run \
     --config=./configs/${RUN_CONFIG} \
     --logdir=$LOGDIR \

bin/train_multi_channels.sh

@@ -4,12 +4,17 @@ export CUDA_VISIBLE_DEVICES=2,3
 RUN_CONFIG=config.yml
 
 
-for channels in [1,2,3,5] [1,2,3,6] [1,2,4,5] [1,2,4,6] [1,2,5,6] [1,3,4,5] [1,3,4,6] [1,3,5,6] [1,4,5,6] [2,3,4,5] [2,3,4,6] [2,3,5,6] [2,4,5,6] [3,4,5,6]; do
-    LOGDIR=/raid/bac/kaggle/logs/recursion_cell/search_channels/$channels/se_resnext50_32x4d/
-    catalyst-dl run \
-        --config=./configs/${RUN_CONFIG} \
-        --logdir=$LOGDIR \
-        --out_dir=$LOGDIR:str \
-        --stages/data_params/channels=$channels:list \
-        --verbose
+# for channels in [1,2,3,5] [1,2,3,6] [1,2,4,5] [1,2,4,6] [1,2,5,6] [1,3,4,5] [1,3,4,6] [1,3,5,6] [1,4,5,6] [2,3,4,5] [2,3,4,6] [2,3,5,6] [2,4,5,6] [3,4,5,6]; do
+for channels in [1,2,3,4]; do
+    for fold in 1 2 3; do
+        LOGDIR=/raid/bac/kaggle/logs/recursion_cell/search_channels/fold_$fold/$channels/se_resnext50_32x4d/
+        catalyst-dl run \
+            --config=./configs/${RUN_CONFIG} \
+            --logdir=$LOGDIR \
+            --out_dir=$LOGDIR:str \
+            --stages/data_params/channels=$channels:list \
+            --stages/data_params/train_csv=./csv/train_$fold.csv:str \
+            --stages/data_params/valid_csv=./csv/valid_$fold.csv:str \
+            --verbose
+    done
 done

configs/config.yml

@@ -22,18 +22,6 @@ stages:
 #    criterion: CrossEntropyLoss
     criterion: LabelSmoothingCrossEntropy
 
-  optimizer_params:
-    optimizer: Nadam
-    lr: 0.0001
-#    weight_decay: 0.0001
-
-  scheduler_params:
-    scheduler: OneCycleLR
-    num_steps: &num_epochs 30
-    lr_range: [0.0005, 0.00001]
-    warmup_steps: 5
-    momentum_range: [0.85, 0.95]
-
   data_params:
     batch_size: 64
     num_workers: 8
@@ -46,14 +34,23 @@ stages:
     sites: [1]
     channels: [1, 2, 3, 4]
 
-  stage1:
+  stage0:
+
+    optimizer_params:
+      optimizer: Nadam
+      lr: 0.001
+
+    scheduler_params:
+      scheduler: MultiStepLR
+      milestones: [10]
+      gamma: 0.3
 
     state_params:
-      num_epochs: *num_epochs
+      num_epochs: 2
 
     callbacks_params: &callback_params
       loss:
-#        callback: CriterionCallback
+        #        callback: CriterionCallback
         callback: LabelSmoothCriterionCallback
       optimizer:
         callback: OptimizerCallback
@@ -67,14 +64,20 @@ stages:
       saver:
         callback: CheckpointCallback
 
-#  stage2:
-#
-#    state_params:
-#      num_epochs: 9
-#
-#    optimizer_params:
-#      optimizer: Adam
-#      lr: 0.0001
-#      weight_decay: 0.0001
-#
-#    callbacks_params: *callback_params
+  stage1:
+
+    optimizer_params:
+      optimizer: Nadam
+      lr: 0.0001
+
+    scheduler_params:
+      scheduler: OneCycleLR
+      num_steps: &num_epochs 30
+      lr_range: [0.0005, 0.00001]
+      warmup_steps: 5
+      momentum_range: [0.85, 0.95]
+
+    state_params:
+      num_epochs: *num_epochs
+
+    callbacks_params: *callback_params

src/augmentation.py

@@ -23,6 +23,7 @@ def train_aug(image_size=224):
         RandomRotate90(),
         Flip(),
         Transpose(),
+        IAAAffine(shear=(-10, 10)),
         # OneOf(
         #     aug_list
         # ),
@@ -36,3 +37,21 @@ def valid_aug(image_size=224):
         Resize(image_size, image_size)
         # Normalize(),
     ], p=1)
+
+
+def test_tta(image_size):
+    test_dict = {
+        'normal': Compose([
+            Resize(image_size, image_size)
+        ]),
+        # 'hflip': Compose([
+        #     HorizontalFlip(p=1),
+        #     Resize(image_size, image_size),
+        # ], p=1),
+        # 'rot90': Compose([
+        #     Rotate(limit=(90, 90), p=1),
+        #     Resize(image_size, image_size),
+        # ], p=1),
+    }
+
+    return test_dict

src/dataset.py

@@ -187,6 +187,7 @@ def __init__(self,
                  ):
         print("Channels ", channels)
         print("sites ", sites)
+        print(csv_file)
         df = pd.read_csv(csv_file, nrows=None)
         self.pixel_stat = pd.read_csv(os.path.join(root, "pixel_stats.csv"))
         self.stat_dict = {}

src/experiment.py

@@ -21,6 +21,15 @@ def _postprocess_model_for_stage(self, stage: str, model: nn.Module):
         if isinstance(model, torch.nn.DataParallel):
             model_ = model_.module
 
+        if stage == "stage0":
+            for param in model_._features.parameters():
+                param.requires_grad = False
+            print("Freeze backbone model !!!")
+        else:
+            for param in model_._features.parameters():
+                param.requires_grad = True
+            print("Unfreeze backbone model !!!")
+
         return model_
 
     def get_datasets(self, stage: str, **kwargs):

src/inference.py

@@ -25,7 +25,7 @@ def predict(model, loader):
         for dct in tqdm(loader, total=len(loader)):
             images = dct['images'].to(device)
             pred = model(images)
-            pred = Ftorch.softmax(pred)
+            # pred = Ftorch.softmax(pred)
             pred = pred.detach().cpu().numpy()
             preds.append(pred)
 
@@ -38,56 +38,71 @@ def predict_all():
     # test_csv = './csv/valid_0.csv'
     model_name = 'se_resnext50_32x4d'
 
-    for channel_str in ["[1,2,4,5]", "[1,2,3,5]", "[1,2,5,6]", "[1,3,4,5]"]:
-        experiment = 'c1234_s1_smooth_nadam_rndsite_64'
+    tta_dict = test_tta(image_size=512)
+
+    for channel_str in [
+        "[1,2,3,4]", "[1,2,3,5]", "[1,2,3,6]",
+        "[1,2,4,5]", "[1,2,4,6]", "[1,2,5,6]",
+        "[1,3,4,5]", "[1,3,4,6]", "[1,3,5,6]",
+        "[1,4,5,6]", "[2,3,4,5]", "[2,3,4,6]",
+        "[2,3,5,6]", "[2,4,5,6]", "[3,4,5,6]"
+    ]:
 
         log_dir = f"/raid/bac/kaggle/logs/recursion_cell/search_channels/{channel_str}/{model_name}/"
         root = "/raid/data/kaggle/recursion-cellular-image-classification/"
         sites = [1]
         channels = [int(i) for i in channel_str[1:-1].split(',')]
 
-        preds = []
-        model = cell_senet(
-            model_name="se_resnext50_32x4d",
-            num_classes=1108,
-            n_channels=len(channels) * len(sites)
-        )
-
-        checkpoint = f"{log_dir}/checkpoints/best.pth"
-        checkpoint = torch.load(checkpoint)
-        model.load_state_dict(checkpoint['model_state_dict'])
-        model = model.to(device)
-        model = nn.DataParallel(model)
-
-        for site in [1, 2]:
-            # Dataset
-            dataset = RecursionCellularSite(
-                csv_file=test_csv,
-                root=root,
-                transform=valid_aug(512),
-                mode='test',
-                sites=[site],
-                channels=channels
-            )
+        log_dir = log_dir.replace('[', '[[]')
 
-            loader = DataLoader(
-                dataset=dataset,
-                batch_size=8,
-                shuffle=False,
-                num_workers=4,
+        all_checkpoints = glob.glob(f"{log_dir}/checkpoints/stage1.*")
+        for ckp_num, ckp in enumerate(all_checkpoints):
+            model = cell_senet(
+                model_name="se_resnext50_32x4d",
+                num_classes=1108,
+                n_channels=len(channels) * len(sites)
             )
 
-            pred = predict(model, loader)
-            preds.append(pred)
-
-        preds = np.asarray(preds).mean(axis=0)
-        all_preds = np.argmax(preds, axis=1)
-        df = pd.read_csv(test_csv)
-        submission = df.copy()
-        submission['sirna'] = all_preds.astype(int)
-        os.makedirs("prediction", exist_ok=True)
-        submission.to_csv(f'./prediction/{model_name}_{channel_str}.csv', index=False, columns=['id_code', 'sirna'])
-        np.save(f"./prediction/{model_name}_{channel_str}.npy", preds)
+            checkpoint = torch.load(ckp)
+            model.load_state_dict(checkpoint['model_state_dict'])
+            model = model.to(device)
+            # model = nn.DataParallel(model)
+
+            for tta_key, tta_value in tta_dict.items():
+                print("*" * 50)
+                print(f"checkpoint: {ckp}")
+                print(f"Channel: {channel_str}")
+                print(f"TTA: {tta_key}")
+                preds = []
+                for site in [1, 2]:
+                    # Dataset
+                    dataset = RecursionCellularSite(
+                        csv_file=test_csv,
+                        root=root,
+                        transform=tta_value,
+                        mode='test',
+                        sites=[site],
+                        channels=channels
+                    )
+
+                    loader = DataLoader(
+                        dataset=dataset,
+                        batch_size=128,
+                        shuffle=False,
+                        num_workers=8,
+                    )
+
+                    pred = predict(model, loader)
+                    preds.append(pred)
+
+                preds = np.asarray(preds).mean(axis=0)
+                all_preds = np.argmax(preds, axis=1)
+                df = pd.read_csv(test_csv)
+                submission = df.copy()
+                submission['sirna'] = all_preds.astype(int)
+                os.makedirs("./prediction/fold_0/", exist_ok=True)
+                submission.to_csv(f'./prediction/fold_0/{model_name}_{channel_str}_ckp{ckp_num}_tta_{tta_key}_test.csv', index=False, columns=['id_code', 'sirna'])
+                np.save(f"./prediction/fold_0/{model_name}_{channel_str}_ckp{ckp_num}_tta_{tta_key}_test.npy", preds)
 
 
 if __name__ == '__main__':

src/make_submission.py

@@ -37,8 +37,64 @@ def predict_all():
     test_csv = '/raid/data/kaggle/recursion-cellular-image-classification/test.csv'
     # test_csv = './csv/valid_0.csv'
 
-    experiment = 'c1234_s1_smooth_nadam_rndsite_64'
+    for fold in [0, 1, 2, 3]:
+        model_name = 'se_resnext50_32x4d'
+
+        log_dir = f"/raid/bac/kaggle/logs/recursion_cell/search_channels/fold_{fold}/[1,2,3,4]/se_resnext50_32x4d/"
+        root = "/raid/data/kaggle/recursion-cellular-image-classification/"
+        sites = [1]
+        channels = [1,2,3,4]
+
+        preds = []
+        model = cell_senet(
+            model_name="se_resnext50_32x4d",
+            num_classes=1108,
+            n_channels=len(channels) * len(sites)
+        )
+
+        checkpoint = f"{log_dir}/checkpoints/best.pth"
+        checkpoint = torch.load(checkpoint)
+        model.load_state_dict(checkpoint['model_state_dict'])
+        model = model.to(device)
+        model = nn.DataParallel(model)
+
+        for site in [1, 2]:
+            # Dataset
+            dataset = RecursionCellularSite(
+                csv_file=test_csv,
+                root=root,
+                transform=valid_aug(512),
+                mode='test',
+                sites=[site],
+                channels=channels
+            )
+
+            loader = DataLoader(
+                dataset=dataset,
+                batch_size=128,
+                shuffle=False,
+                num_workers=4,
+            )
+
+            pred = predict(model, loader)
+            preds.append(pred)
+
+        preds = np.asarray(preds).mean(axis=0)
+        all_preds = np.argmax(preds, axis=1)
+        df = pd.read_csv(test_csv)
+        submission = df.copy()
+        submission['sirna'] = all_preds.astype(int)
+        os.makedirs("predict_folds/", exist_ok=True)
+        submission.to_csv(f'./predict_folds/{model_name}_[1,2,3,4]_{fold}.csv', index=False, columns=['id_code', 'sirna'])
+        np.save(f"./predict_folds/{model_name}_[1,2,3,4]_{fold}.npy", preds)
+
+
+def predict_one():
+    test_csv = '/raid/data/kaggle/recursion-cellular-image-classification/test.csv'
+    # test_csv = './csv/valid_0.csv'
+
     model_name = 'se_resnext50_32x4d'
+    experiment = "c1234_s1_affine_warmup"
 
     log_dir = f"/raid/bac/kaggle/logs/recursion_cell/test/{experiment}/{model_name}/"
     root = "/raid/data/kaggle/recursion-cellular-image-classification/"
@@ -84,10 +140,11 @@ def predict_all():
     df = pd.read_csv(test_csv)
     submission = df.copy()
     submission['sirna'] = all_preds.astype(int)
-    os.makedirs("submission", exist_ok=True)
-    submission.to_csv(f'./submission/{model_name}_{experiment}_3ckps.csv', index=False, columns=['id_code', 'sirna'])
-    np.save(f"./submission/{model_name}_{experiment}_3ckps.npy", preds)
+    os.makedirs("submission/", exist_ok=True)
+    submission.to_csv(f'./submission/{model_name}_{experiment}.csv', index=False, columns=['id_code', 'sirna'])
+    np.save(f"./submission/{model_name}_{experiment}.npy", preds)
 
 
 if __name__ == '__main__':
-    predict_all()
+    # predict_all()
+    predict_one()