diff --git a/.github/workflows/llm_bench-python.yml b/.github/workflows/llm_bench-python.yml index bc318afca..fab31e573 100644 --- a/.github/workflows/llm_bench-python.yml +++ b/.github/workflows/llm_bench-python.yml @@ -40,6 +40,7 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest black pip install -r ${{ env.LLM_BENCH_PYPATH }}/requirements.txt + pip install openvino-nightly - name: Lint with flake8 run: | @@ -55,8 +56,10 @@ jobs: with: name: llm.bench_diff path: llm.bench_diff.diff - - name: Run llm_bench test on linux - run: llm_bench/python/llm_run_on_linux.sh + - name: Test bloomz-560m on Linux + run: llm_bench/python/llm_run_bloomz_560m_on_linux.sh + - name: Test tiny-stable-diffusion on Linux + run: llm_bench/python/llm_run_stable_diffusion_on_linux.sh stateful: runs-on: ubuntu-20.04 steps: diff --git a/llm_bench/python/llm_run_bloomz_560m_on_linux.sh b/llm_bench/python/llm_run_bloomz_560m_on_linux.sh new file mode 100755 index 000000000..27bdc61cf --- /dev/null +++ b/llm_bench/python/llm_run_bloomz_560m_on_linux.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +cat /proc/cpuinfo | grep 'model name' | uniq +grep MemTotal /proc/meminfo + +git lfs install +export GIT_LFS_SKIP_SMUDGE=0 +echo "Download bloomz-560m" +git_clone_bloomz_560m="git clone https://huggingface.co/bigscience/bloomz-560m" +echo ${git_clone_bloomz_560m} +eval ${git_clone_bloomz_560m} +wait + +original_dir="bloomz-560m" +dst_dir="./ov_models/bloomz-560m" + +convert_model="python ./llm_bench/python/convert.py --model_id ${original_dir} --output_dir ${dst_dir} --precision FP16 --stateful" +echo ${convert_model} +eval ${convert_model} +wait + +bemchmarking="python ./llm_bench/python/benchmark.py -m ${dst_dir}/pytorch/dldt/FP16/ -d cpu -n 1" +echo ${bemchmarking} +eval ${bemchmarking} + +rm -rf ${original_dir} +rm -rf ${dst_dir} \ No newline at end of file diff --git a/llm_bench/python/llm_run_on_linux.sh b/llm_bench/python/llm_run_on_linux.sh deleted file mode 100755 index 209f0faf2..000000000 --- a/llm_bench/python/llm_run_on_linux.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# Copyright (C) 2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -git lfs install -export GIT_LFS_SKIP_SMUDGE=0 -git_clone_bloomz_560m="git clone https://huggingface.co/bigscience/bloomz-560m" -echo ${git_clone_bloomz_560m} -eval ${git_clone_bloomz_560m} -wait - -convert_model="python ./llm_bench/python/convert.py --model_id bloomz-560m/ --output_dir ./ov_models/bloomz-560m --precision FP16" -echo ${convert_model} -eval ${convert_model} -wait - -bemchmarking="python ./llm_bench/python/benchmark.py -m ./ov_models/bloomz-560m/pytorch/dldt/FP16/ -d cpu -n 1" -echo ${bemchmarking} -eval ${bemchmarking} \ No newline at end of file diff --git a/llm_bench/python/llm_run_stable_diffusion_on_linux.sh b/llm_bench/python/llm_run_stable_diffusion_on_linux.sh new file mode 100755 index 000000000..9657dab5b --- /dev/null +++ b/llm_bench/python/llm_run_stable_diffusion_on_linux.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (C) 2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +cat /proc/cpuinfo | grep 'model name' | uniq +grep MemTotal /proc/meminfo + +git lfs install +export GIT_LFS_SKIP_SMUDGE=0 +echo "Download tiny-sd" +git_clone_stable_diff="git clone https://huggingface.co/segmind/tiny-sd" +echo ${git_clone_stable_diff} +eval ${git_clone_stable_diff} +wait + +original_dir="tiny-sd" +dst_dir="./ov_models/tiny-sd" + +convert_model="python ./llm_bench/python/convert.py --model_id ${original_dir} --output_dir ${dst_dir} --precision FP16 --stateful" +echo ${convert_model} +eval ${convert_model} +wait + +bemchmarking="python ./llm_bench/python/benchmark.py -m ${dst_dir}/pytorch/dldt/FP16/ -pf ./llm_bench/python/prompts/stable-diffusion.jsonl -d cpu -n 1" +echo ${bemchmarking} +eval ${bemchmarking} + +rm -rf ${original_dir} +rm -rf ${dst_dir} \ No newline at end of file diff --git a/llm_bench/python/prompts/stable-diffusion.jsonl b/llm_bench/python/prompts/stable-diffusion.jsonl new file mode 100644 index 000000000..59c23064a --- /dev/null +++ b/llm_bench/python/prompts/stable-diffusion.jsonl @@ -0,0 +1 @@ +{"steps":"30", "width":"256", "height":"256", "guidance_scale":"1.0", "prompt": "side profile centered painted portrait, Gandhi rolling a blunt, Gloomhaven, matte painting concept art, art nouveau, 8K HD Resolution, beautifully background"} \ No newline at end of file