Reproducibility grid and ablation study

conf/config.yaml

@@ -184,7 +184,7 @@ dora:
 slurm:
   time: 4320
   constraint: volta32gb
-  setup: ['module load cuda/11.0 cudnn/v8.0.3.33-cuda.11.0 NCCL/2.8.3-1-cuda.11.0']
+  setup: ['module load cudnn/v8.4.1.50-cuda.11.6 NCCL/2.11.4-6-cuda.11.6 cuda/11.6']
 
 # Hydra config
 hydra:

demucs/grids/_explorers.py

@@ -8,7 +8,7 @@
 
 
 class MyExplorer(Explorer):
-    test_metrics = ['nsdr']
+    test_metrics = ['nsdr', 'sdr_med']
 
     def get_grid_metrics(self):
         """Return the metrics that should be displayed in the tracking table.

demucs/grids/repro.py

@@ -0,0 +1,47 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Easier training for reproducibility
+"""
+
+from ._explorers import MyExplorer
+
+
+@MyExplorer
+def explorer(launcher):
+    launcher.slurm_(
+        gpus=8,
+        time=3 * 24 * 60,
+        partition='devlab,learnlab')
+
+    launcher.bind_({'ema.epoch': [0.9, 0.95]})
+    launcher.bind_({'ema.batch': [0.9995, 0.9999]})
+    launcher.bind_({'epochs': 600})
+
+    base = {'model': 'demucs', 'demucs.dconv_mode': 0, 'demucs.gelu': False,
+            'demucs.lstm_layers': 2}
+    newt = {'model': 'demucs', 'demucs.normalize': True}
+    hdem = {'model': 'hdemucs'}
+    svd = {'svd.penalty': 1e-5, 'svd': 'base2'}
+
+    with launcher.job_array():
+        for model in [base, newt, hdem]:
+            sub = launcher.bind(model)
+            if model is base:
+                sub(epochs=360)
+                continue
+            sub(svd)
+            sub(svd, seed=43)
+            if model == newt:
+                # Ablation study
+                sub()
+                abl = sub.bind(svd)
+                abl({'ema.epoch': [], 'ema.batch': []})
+                abl({'demucs.dconv_lstm': 10})
+                abl({'demucs.dconv_attn': 10})
+                abl({'demucs.dconv_attn': 10, 'demucs.dconv_lstm': 10, 'demucs.lstm_layers': 2})
+                abl({'demucs.dconv_mode': 0})
+                abl({'demucs.gelu': False})

demucs/grids/repro_ft.py

@@ -0,0 +1,46 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Fine tuning experiments
+"""
+
+from ._explorers import MyExplorer
+from ..train import main
+
+
+@MyExplorer
+def explorer(launcher):
+    launcher.slurm_(
+        gpus=8,
+        time=300,
+        partition='devlab,learnlab')
+
+    # Mus
+    launcher.slurm_(constraint='volta32gb')
+
+    grid = "repro"
+    folder = main.dora.dir / "grids" / grid
+
+    for sig in folder.iterdir():
+        if not sig.is_symlink():
+            continue
+        xp = main.get_xp_from_sig(sig)
+        xp.link.load()
+        if len(xp.link.history) != xp.cfg.epochs:
+            continue
+        sub = launcher.bind(xp.argv, [f'continue_from="{xp.sig}"'])
+        sub.bind_({'ema.epoch': [0.9, 0.95], 'ema.batch': [0.9995, 0.9999]})
+        sub.bind_({'test.every': 1, 'test.sdr': True, 'epochs': 4})
+        sub.bind_({'dset.segment': 28, 'dset.shift': 2})
+        sub.bind_({'batch_size': 32})
+        auto = {'dset': 'auto_mus'}
+        auto.update({'augment.remix.proba': 0, 'augment.scale.proba': 0,
+                     'augment.shift_same': True})
+        sub.bind_(auto)
+        sub.bind_({'batch_size': 16})
+        sub.bind_({'optim.lr': 1e-4})
+        sub.bind_({'model_segment': 44})
+        sub()

demucs/remote/files.txt

@@ -14,3 +14,7 @@ b72baf4e-8778635e.th
 464b36d7-e5a9386e.th
 7fd6ef75-a905dd85.th
 83fc094f-4a16d450.th
+1ef250f1-592467ce.th
+902315c2-b39ce9c9.th
+9a6b4851-03af0aa6.th
+fa0cb7f9-100d8bf4.th

demucs/remote/repro_mdx_a.yaml

@@ -0,0 +1,2 @@
+models: ['9a6b4851', '1ef250f1', 'fa0cb7f9', '902315c2']
+segment: 44

demucs/remote/repro_mdx_a_hybrid_only.yaml

@@ -0,0 +1,2 @@
+models: ['fa0cb7f9', '902315c2', 'fa0cb7f9', '902315c2']
+segment: 44

demucs/remote/repro_mdx_a_time_only.yaml

@@ -0,0 +1,2 @@
+models: ['9a6b4851', '9a6b4851', '1ef250f1', '1ef250f1']
+segment: 44