forked from mistralai/mistral-inference
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Dockerfile and build instructions
- Loading branch information
Showing
4 changed files
with
59 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
* | ||
!entrypoint.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
FROM --platform=amd64 nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 as base | ||
|
||
ARG MAX_JOBS | ||
|
||
WORKDIR /workspace | ||
|
||
RUN apt update && \ | ||
apt install -y python3-pip python3-packaging \ | ||
git ninja-build && \ | ||
pip3 install -U pip | ||
|
||
# Tweak this list to reduce build time | ||
# https://developer.nvidia.com/cuda-gpus | ||
ENV TORCH_CUDA_ARCH_LIST "7.0;7.2;7.5;8.0;8.6;8.9;9.0" | ||
|
||
# We have to manually install Torch otherwise apex & xformers won't build | ||
RUN pip3 install "torch>=2.0.0" | ||
|
||
# This build is slow but NVIDIA does not provide binaries. Increase MAX_JOBS as needed. | ||
RUN git clone https://github.com/NVIDIA/apex && \ | ||
cd apex && \ | ||
sed -i '/check_cuda_torch_binary_vs_bare_metal(CUDA_HOME)/d' setup.py && \ | ||
python3 setup.py install --cpp_ext --cuda_ext | ||
|
||
RUN pip3 install "xformers==0.0.22" "transformers==4.33.3" "vllm==0.2.0" | ||
|
||
# Fastchat is not yet released, so use our own commit | ||
RUN pip3 install "fschat[model_worker] @ git+https://github.com/mistralai/FastChat-release.git@bde1118ed8df8c42fe98076424f60505465d4265#egg=FastChat" | ||
|
||
COPY entrypoint.sh . | ||
|
||
RUN chmod +x /workspace/entrypoint.sh | ||
|
||
ENTRYPOINT ["/workspace/entrypoint.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#!/bin/bash | ||
|
||
if [[ ! -z "${HF_TOKEN}" ]]; then | ||
echo "The HF_TOKEN environment variable set, logging to Hugging Face." | ||
python3 -c "import huggingface_hub; huggingface_hub.login('${HF_TOKEN}')" | ||
else | ||
echo "The HF_TOKEN environment variable is not set or empty, not logging to Hugging Face." | ||
fi | ||
|
||
# Run the provided command | ||
exec python3 -u -m vllm.entrypoints.openai.api_server "$@" |