sweeps

the-full-stack · Nov 16, 2019 · fec8dab · fec8dab
1 parent de23fd8
commit fec8dab
Show file tree

Hide file tree

Showing 13 changed files with 124 additions and 103 deletions.
diff --git a/lab3/readme.md b/lab3/readme.md
@@ -25,6 +25,8 @@ cd lab3/
 
 ### Let's get started with W&B!
 
+NOTE: These instructions are optional if you're working in the pre-configured Jupyter hub.
+
 ```
 pipenv run wandb init
 ```
@@ -46,7 +48,7 @@ Select `fsdl-text-recognizer-project`.
 
 How to implement W&B in training code?
 
-Look at `training/run_experiment.py`.
+Look at `training/run_experiment.py` and `training/util.py`
 
 ### Your first W&B experiment
 

diff --git a/lab3/training/run_sweep.py b/lab3/training/run_sweep.py
diff --git a/lab4/readme.md b/lab4/readme.md
@@ -39,7 +39,7 @@ Training longer will keep improving: the same settings get to 60% accuracy in 40
 
 For the rest of the lab, let's play around with different things and see if we can improve performance quickly.
 
-You can see all of our training runs here: https://app.wandb.ai/fsdl/fsdl-text-recognizer-project
+You can see all of our training runs here: https://app.wandb.ai/fsdl/fsdl-text-recognizer-nov16
 Feel free to peek in on your neighbors!
 
 If you commit and push your code changes, then the run will also be linked to the exact code your ran, which you will be able to review months later if necessary.

diff --git a/lab4/training/run_sweep.py b/lab4/training/run_sweep.py
@@ -1,17 +1,19 @@
 # pylint: skip-file
 import os
+import signal
+import subprocess
 import sys
 import json
 from typing import Tuple
 from ast import literal_eval
 
 default_config = {
-    "dataset": "EmnistLinesDataset",
+    "dataset": "IamLinesDataset",
     "dataset_args": {
         "max_overlap": 0.4,
     },
-    "model": "LineModel",
-    "network": "line_cnn_all_conv",
+    "model": "LineModelCtc",
+    "network": "line_lstm_ctc",
     "network_args": {
         "window_width": 14,
         "window_stride": 7
@@ -65,7 +67,9 @@ def args_to_json(default_config: dict, preserve_args: list = ["gpu", "save"]) ->
 
 
 if __name__ == "__main__":
-    for key, val in os.environ.items():
-        os.putenv(key, val)
     config, args = args_to_json(default_config)
-    os.system("python training/run_experiment.py {} '{}'".format((" ").join(args), json.dumps(config)))  # nosec
+    env = {k: v for k, v in os.environ.items() if k not in ("WANDB_PROGRAM", "WANDB_ARGS")}
+    run = subprocess.Popen(["python", "training/run_experiment.py", *args, json.dumps(config)],
+                           env=env, preexec_fn=os.setsid)  # nosec
+    signal.signal(signal.SIGTERM, lambda *args: run.terminate())
+    run.wait()
diff --git a/lab4/training/sweep.yaml b/lab4/training/sweep.yaml
@@ -0,0 +1,14 @@
+program: training/run_sweep.py
+method: grid
+metric:
+  name: val_loss
+  goal: minimize
+parameters:
+  dataset_args.max_overlap:
+    values: [0.1, 0.4, 0.7]
+  network_args.window_width:
+    values: [10, 20]
+  network_args.window_stride:
+    values: [5, 10]
+  train_args.batch_size:
+    values: [32, 64, 128, 256]
diff --git a/lab5/training/run_sweep.py b/lab5/training/run_sweep.py
@@ -1,17 +1,19 @@
 # pylint: skip-file
 import os
+import signal
+import subprocess
 import sys
 import json
 from typing import Tuple
 from ast import literal_eval
 
 default_config = {
-    "dataset": "EmnistLinesDataset",
+    "dataset": "IamLinesDataset",
     "dataset_args": {
         "max_overlap": 0.4,
     },
-    "model": "LineModel",
-    "network": "line_cnn_all_conv",
+    "model": "LineModelCtc",
+    "network": "line_lstm_ctc",
     "network_args": {
         "window_width": 14,
         "window_stride": 7
@@ -65,7 +67,9 @@ def args_to_json(default_config: dict, preserve_args: list = ["gpu", "save"]) ->
 
 
 if __name__ == "__main__":
-    for key, val in os.environ.items():
-        os.putenv(key, val)
     config, args = args_to_json(default_config)
-    os.system("python training/run_experiment.py {} '{}'".format((" ").join(args), json.dumps(config)))  # nosec
+    env = {k: v for k, v in os.environ.items() if k not in ("WANDB_PROGRAM", "WANDB_ARGS")}
+    run = subprocess.Popen(["python", "training/run_experiment.py", *args, json.dumps(config)],
+                           env=env, preexec_fn=os.setsid)  # nosec
+    signal.signal(signal.SIGTERM, lambda *args: run.terminate())
+    run.wait()
diff --git a/lab5/training/sweep.yaml b/lab5/training/sweep.yaml
@@ -0,0 +1,14 @@
+program: training/run_sweep.py
+method: grid
+metric:
+  name: val_loss
+  goal: minimize
+parameters:
+  dataset_args.max_overlap:
+    values: [0.1, 0.4, 0.7]
+  network_args.window_width:
+    values: [10, 20]
+  network_args.window_stride:
+    values: [5, 10]
+  train_args.batch_size:
+    values: [32, 64, 128, 256]
diff --git a/lab6/training/run_sweep.py b/lab6/training/run_sweep.py
@@ -1,17 +1,19 @@
 # pylint: skip-file
 import os
+import signal
+import subprocess
 import sys
 import json
 from typing import Tuple
 from ast import literal_eval
 
 default_config = {
-    "dataset": "EmnistLinesDataset",
+    "dataset": "IamLinesDataset",
     "dataset_args": {
         "max_overlap": 0.4,
     },
-    "model": "LineModel",
-    "network": "line_cnn_all_conv",
+    "model": "LineModelCtc",
+    "network": "line_lstm_ctc",
     "network_args": {
         "window_width": 14,
         "window_stride": 7
@@ -65,7 +67,9 @@ def args_to_json(default_config: dict, preserve_args: list = ["gpu", "save"]) ->
 
 
 if __name__ == "__main__":
-    for key, val in os.environ.items():
-        os.putenv(key, val)
     config, args = args_to_json(default_config)
-    os.system("python training/run_experiment.py {} '{}'".format((" ").join(args), json.dumps(config)))  # nosec
+    env = {k: v for k, v in os.environ.items() if k not in ("WANDB_PROGRAM", "WANDB_ARGS")}
+    run = subprocess.Popen(["python", "training/run_experiment.py", *args, json.dumps(config)],
+                           env=env, preexec_fn=os.setsid)  # nosec
+    signal.signal(signal.SIGTERM, lambda *args: run.terminate())
+    run.wait()
diff --git a/lab6/training/sweep.yaml b/lab6/training/sweep.yaml
@@ -0,0 +1,14 @@
+program: training/run_sweep.py
+method: grid
+metric:
+  name: val_loss
+  goal: minimize
+parameters:
+  dataset_args.max_overlap:
+    values: [0.1, 0.4, 0.7]
+  network_args.window_width:
+    values: [10, 20]
+  network_args.window_stride:
+    values: [5, 10]
+  train_args.batch_size:
+    values: [32, 64, 128, 256]
diff --git a/lab7/training/run_sweep.py b/lab7/training/run_sweep.py
@@ -1,17 +1,19 @@
 # pylint: skip-file
 import os
+import signal
+import subprocess
 import sys
 import json
 from typing import Tuple
 from ast import literal_eval
 
 default_config = {
-    "dataset": "EmnistLinesDataset",
+    "dataset": "IamLinesDataset",
     "dataset_args": {
         "max_overlap": 0.4,
     },
-    "model": "LineModel",
-    "network": "line_cnn_all_conv",
+    "model": "LineModelCtc",
+    "network": "line_lstm_ctc",
     "network_args": {
         "window_width": 14,
         "window_stride": 7
@@ -65,7 +67,9 @@ def args_to_json(default_config: dict, preserve_args: list = ["gpu", "save"]) ->
 
 
 if __name__ == "__main__":
-    for key, val in os.environ.items():
-        os.putenv(key, val)
     config, args = args_to_json(default_config)
-    os.system("python training/run_experiment.py {} '{}'".format((" ").join(args), json.dumps(config)))  # nosec
+    env = {k: v for k, v in os.environ.items() if k not in ("WANDB_PROGRAM", "WANDB_ARGS")}
+    run = subprocess.Popen(["python", "training/run_experiment.py", *args, json.dumps(config)],
+                           env=env, preexec_fn=os.setsid)  # nosec
+    signal.signal(signal.SIGTERM, lambda *args: run.terminate())
+    run.wait()
diff --git a/lab7/training/sweep.yaml b/lab7/training/sweep.yaml
@@ -0,0 +1,14 @@
+program: training/run_sweep.py
+method: grid
+metric:
+  name: val_loss
+  goal: minimize
+parameters:
+  dataset_args.max_overlap:
+    values: [0.1, 0.4, 0.7]
+  network_args.window_width:
+    values: [10, 20]
+  network_args.window_stride:
+    values: [5, 10]
+  train_args.batch_size:
+    values: [32, 64, 128, 256]
diff --git a/lab8/training/run_sweep.py b/lab8/training/run_sweep.py
@@ -1,17 +1,19 @@
 # pylint: skip-file
 import os
+import signal
+import subprocess
 import sys
 import json
 from typing import Tuple
 from ast import literal_eval
 
 default_config = {
-    "dataset": "EmnistLinesDataset",
+    "dataset": "IamLinesDataset",
     "dataset_args": {
         "max_overlap": 0.4,
     },
-    "model": "LineModel",
-    "network": "line_cnn_all_conv",
+    "model": "LineModelCtc",
+    "network": "line_lstm_ctc",
     "network_args": {
         "window_width": 14,
         "window_stride": 7
@@ -65,7 +67,9 @@ def args_to_json(default_config: dict, preserve_args: list = ["gpu", "save"]) ->
 
 
 if __name__ == "__main__":
-    for key, val in os.environ.items():
-        os.putenv(key, val)
     config, args = args_to_json(default_config)
-    os.system("python training/run_experiment.py {} '{}'".format((" ").join(args), json.dumps(config)))  # nosec
+    env = {k: v for k, v in os.environ.items() if k not in ("WANDB_PROGRAM", "WANDB_ARGS")}
+    run = subprocess.Popen(["python", "training/run_experiment.py", *args, json.dumps(config)],
+                           env=env, preexec_fn=os.setsid)  # nosec
+    signal.signal(signal.SIGTERM, lambda *args: run.terminate())
+    run.wait()
diff --git a/lab8/training/sweep.yaml b/lab8/training/sweep.yaml
@@ -0,0 +1,14 @@
+program: training/run_sweep.py
+method: grid
+metric:
+  name: val_loss
+  goal: minimize
+parameters:
+  dataset_args.max_overlap:
+    values: [0.1, 0.4, 0.7]
+  network_args.window_width:
+    values: [10, 20]
+  network_args.window_stride:
+    values: [5, 10]
+  train_args.batch_size:
+    values: [32, 64, 128, 256]