automatically find unused tcp port when mgpu train or test (open-mmla…

…b#224) * automatically find unused tcp port when mgpu train or test * fixbug: set tcp_port before set python args
ChienLiu · Aug 9, 2020 · aecc89c · aecc89c
1 parent 0741976
commit aecc89c
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 3 deletions.
diff --git a/tools/scripts/slurm_test_mgpu.sh b/tools/scripts/slurm_test_mgpu.sh
@@ -9,7 +9,15 @@ PY_ARGS=${@:3}
 JOB_NAME=eval
 SRUN_ARGS=${SRUN_ARGS:-""}
 
-PORT=$(( ( RANDOM % 10000 )  + 10000 ))
+while true
+do
+    PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 ))
+    status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)"
+    if [ "${status}" != "0" ]; then
+        break;
+    fi
+done
+echo $PORT
 
 srun -p ${PARTITION} \
     --job-name=${JOB_NAME} \
@@ -18,5 +26,5 @@ srun -p ${PARTITION} \
     --ntasks-per-node=${GPUS_PER_NODE} \
     --kill-on-bad-exit=1 \
     ${SRUN_ARGS} \
-    python -u test.py --launcher slurm ${PY_ARGS} --tcp_port $PORT
+    python -u test.py --launcher slurm --tcp_port $PORT ${PY_ARGS}
 
diff --git a/tools/scripts/slurm_train.sh b/tools/scripts/slurm_train.sh
@@ -11,7 +11,15 @@ GPUS_PER_NODE=${GPUS_PER_NODE:-8}
 CPUS_PER_TASK=${CPUS_PER_TASK:-5}
 SRUN_ARGS=${SRUN_ARGS:-""}
 
-PORT=$(( ( RANDOM % 10000 )  + 10000 ))
+while true
+do
+    PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 ))
+    status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)"
+    if [ "${status}" != "0" ]; then
+        break;
+    fi
+done
+echo $PORT
 
 srun -p ${PARTITION} \
     --job-name=${JOB_NAME} \