add some modification to support torchbench-cpu well

alibaba · Nov 30, 2022 · 50d7966 · 50d7966
1 parent fa0c827
commit 50d7966
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 6 deletions.
diff --git a/.github/workflows/torchbench.yml b/.github/workflows/torchbench.yml
@@ -70,6 +70,20 @@ jobs:
       remote_dev_docker: ""
     secrets: inherit
 
+  TorchBenchCpuTiny8Threads:
+    if: github.event.schedule != '30 15 * * 5' # daily or dispatch
+    uses: ./.github/workflows/reusable.yml
+    with:
+      name: torch-offcial-benchmark
+      base_image: bladedisc/bladedisc:latest-runtime-torch1.12.0-cpu
+      device: cpu_benchmark
+      dockerfile: docker/cronjobs/Dockerfile.torch.bench
+      extra_envs: -e RELATED_DIFF_PERCENT=3
+      exec_command: bash ./pytorch_blade/benchmark/TorchBench/test_torch_bench.sh cpu tiny 8 "0-7"
+      push_command: ""
+      remote_dev_docker: ""
+    secrets: inherit
+
   TorchBenchCpuPartial:
     if: github.event.schedule == '30 15 * * *'
     uses: ./.github/workflows/reusable.yml
@@ -84,6 +98,20 @@ jobs:
       remote_dev_docker: ""
     secrets: inherit
 
+  TorchBenchCpuPartial8Threads:
+    if: github.event.schedule == '30 15 * * *'
+    uses: ./.github/workflows/reusable.yml
+    with:
+      name: torch-offcial-benchmark
+      base_image: bladedisc/bladedisc:latest-runtime-torch1.12.0-cpu
+      device: cpu_benchmark
+      dockerfile: docker/cronjobs/Dockerfile.torch.bench
+      extra_envs: -e RELATED_DIFF_PERCENT=3
+      exec_command: bash ./pytorch_blade/benchmark/TorchBench/test_torch_bench.sh cpu partial 8 "0-7"
+      push_command: ""
+      remote_dev_docker: ""
+    secrets: inherit
+
   TorchBenchCpuFull:
     if: github.event.schedule == '30 15 * * 5'
     uses: ./.github/workflows/reusable.yml
@@ -97,4 +125,19 @@ jobs:
       exec_command: bash ./pytorch_blade/benchmark/TorchBench/test_torch_bench.sh cpu full
       push_command: ""
       remote_dev_docker: ""
-    secrets: inherit
+    secrets: inherit
+
+  TorchBenchCpuFull8Threads:
+    if: github.event.schedule == '30 15 * * 5'
+    uses: ./.github/workflows/reusable.yml
+    with:
+      name: torch-offcial-benchmark
+      base_image: bladedisc/bladedisc:latest-runtime-torch1.12.0-cpu
+      device: cpu_benchmark
+      timeout_minutes: 4320 # 72h
+      dockerfile: docker/cronjobs/Dockerfile.torch.bench
+      extra_envs: -e RELATED_DIFF_PERCENT=3
+      exec_command: bash ./pytorch_blade/benchmark/TorchBench/test_torch_bench.sh cpu full 8 "0-7"
+      push_command: ""
+      remote_dev_docker: ""
+    secrets: inherit
diff --git a/pytorch_blade/benchmark/TorchBench/requirements_cpu.txt b/pytorch_blade/benchmark/TorchBench/requirements_cpu.txt
@@ -4,4 +4,7 @@ librosa
 torchvision==0.13.0+cpu
 torchaudio==0.12.0+cpu
 torchtext==0.13.0
+onnx
+onnxruntime
+intel_extension_for_pytorch==1.12.0
 torchdynamo
diff --git a/pytorch_blade/benchmark/TorchBench/test_torch_bench.sh b/pytorch_blade/benchmark/TorchBench/test_torch_bench.sh
@@ -34,20 +34,30 @@ ln -s $benchmark_repo_dir torchbenchmark
 # setup benchmark env
 export DISC_EXPERIMENTAL_SPECULATION_TLP_ENHANCE=true \
     DISC_CPU_LARGE_CONCAT_NUM_OPERANDS=4 DISC_CPU_ENABLE_EAGER_TRANSPOSE_FUSION=1 \
-    OMP_NUM_THREADS=1 TORCHBENCH_ATOL=1e-2 TORCHBENCH_RTOL=1e-2
+    TORCHBENCH_ATOL=1e-2 TORCHBENCH_RTOL=1e-2
 
 config_file=blade_$1_$2.yaml
 bench_target=$2
 
 if [ $1 == "cpu" ]
 then
-    # 4 cores
-    export GOMP_CPU_AFFINITY="2-5"
+    binding_cores=0
     results=(eval-cpu-fp32)
+    if [ -z "$3" ]
+    then
+        export OMP_NUM_THREADS=1 GOMP_CPU_AFFINITY=0
+    else
+        # eg: bash test_torch_bench.sh cpu tiny 32 "0-63:2"
+        export OMP_NUM_THREADS=$3 GOMP_CPU_AFFINITY=$4
+        binding_cores=$4
+    fi
+    taskset -c $binding_cores python3 torchbenchmark/.github/scripts/run-config.py \
+            -c $config_file -b ./torchbenchmark/ --output-dir .
 else
     results=(eval-cuda-fp32 eval-cuda-fp16)
+    python3 torchbenchmark/.github/scripts/run-config.py \
+            -c $config_file -b ./torchbenchmark/ --output-dir .
 fi
-python3 torchbenchmark/.github/scripts/run-config.py -c $config_file -b ./torchbenchmark/ --output-dir .
 
 # results
 date_str=$(date '+%Y%m%d-%H')
@@ -58,7 +68,11 @@ for result in ${results[@]}
 do
     cat ${result}/summary.csv
     curl ${oss_link}/TorchBench/baseline/${result}_${bench_target}.csv -o $result.csv
-    /disc/scripts/ci/ossutil cp -r ${script_dir}/${result} ${oss_dir}/${result}
+    if [ $1 == "cpu" ] && [ -n $3 ]; then
+        /disc/scripts/ci/ossutil cp -r ${script_dir}/${result} ${oss_dir}/${result}-$3
+    else
+        /disc/scripts/ci/ossutil cp -r ${script_dir}/${result} ${oss_dir}/${result}
+    fi
 done
 
 # performance anaysis