From f02bd578b7f615e92fb3b63e7be96415b70505cf Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Thu, 11 Apr 2024 10:54:32 +0200 Subject: [PATCH 01/28] Fix dummy cache allocation --- src/petals/server/throughput.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/petals/server/throughput.py b/src/petals/server/throughput.py index d9471790f..2302d3ccf 100644 --- a/src/petals/server/throughput.py +++ b/src/petals/server/throughput.py @@ -206,7 +206,7 @@ def measure_compute_rps( block = block.to(dtype) block = convert_block(block, 0, config, tensor_parallel_devices, device, quant_type=quant_type, freeze=True) - cache = (DUMMY_KEY_PAST.to(dtype), DUMMY_KEY_PAST.to(dtype)) + cache = (DUMMY_KEY_PAST.to(dtype).to(device), DUMMY_KEY_PAST.to(dtype).to(device)) elapsed = 0 dummy_input = torch.randn(1, n_tokens, config.hidden_size, device=device, dtype=dtype) From e5dddfe0b65de0937d179ff0f200515c3e35ab8e Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Thu, 11 Apr 2024 11:37:27 +0200 Subject: [PATCH 02/28] Try mps device selecting --- .github/workflows/run-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d6316d48b..32baf1141 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -14,8 +14,8 @@ jobs: - { model: 'bigscience/bloom-560m', os: 'ubuntu', python-version: '3.11' } - { model: 'Maykeye/TinyLLama-v0', os: 'ubuntu', python-version: '3.8' } - { model: 'Maykeye/TinyLLama-v0', os: 'ubuntu', python-version: '3.11' } - - { model: 'Maykeye/TinyLLama-v0', os: 'macos', python-version: '3.10' } - - { model: 'Maykeye/TinyLLama-v0', os: 'macos', python-version: '3.11' } + - { model: 'Maykeye/TinyLLama-v0', os: 'macos-14', python-version: '3.10' } + - { model: 'Maykeye/TinyLLama-v0', os: 'macos-14', python-version: '3.11' } - { model: 'artek0chumak/TestMixtral', os: 'ubuntu', python-version: '3.8' } - { model: 'artek0chumak/TestMixtral', os: 'ubuntu', python-version: '3.11' } fail-fast: false From 0ca54a5e76c0fa73522198cbeb742ad1e0e8d948 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Thu, 11 Apr 2024 11:41:16 +0200 Subject: [PATCH 03/28] Rechain reloc --- .github/workflows/run-tests.yaml | 4 ++-- src/petals/server/throughput.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 32baf1141..d6316d48b 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -14,8 +14,8 @@ jobs: - { model: 'bigscience/bloom-560m', os: 'ubuntu', python-version: '3.11' } - { model: 'Maykeye/TinyLLama-v0', os: 'ubuntu', python-version: '3.8' } - { model: 'Maykeye/TinyLLama-v0', os: 'ubuntu', python-version: '3.11' } - - { model: 'Maykeye/TinyLLama-v0', os: 'macos-14', python-version: '3.10' } - - { model: 'Maykeye/TinyLLama-v0', os: 'macos-14', python-version: '3.11' } + - { model: 'Maykeye/TinyLLama-v0', os: 'macos', python-version: '3.10' } + - { model: 'Maykeye/TinyLLama-v0', os: 'macos', python-version: '3.11' } - { model: 'artek0chumak/TestMixtral', os: 'ubuntu', python-version: '3.8' } - { model: 'artek0chumak/TestMixtral', os: 'ubuntu', python-version: '3.11' } fail-fast: false diff --git a/src/petals/server/throughput.py b/src/petals/server/throughput.py index 2302d3ccf..c30d2877a 100644 --- a/src/petals/server/throughput.py +++ b/src/petals/server/throughput.py @@ -206,7 +206,7 @@ def measure_compute_rps( block = block.to(dtype) block = convert_block(block, 0, config, tensor_parallel_devices, device, quant_type=quant_type, freeze=True) - cache = (DUMMY_KEY_PAST.to(dtype).to(device), DUMMY_KEY_PAST.to(dtype).to(device)) + cache = (DUMMY_KEY_PAST.to(dtype=dtype, device=device), DUMMY_KEY_PAST.to(dtype=dtype, device=device)) elapsed = 0 dummy_input = torch.randn(1, n_tokens, config.hidden_size, device=device, dtype=dtype) From c76e447ac7ae0999a8c7f885184d4aa4f0cd8b4a Mon Sep 17 00:00:00 2001 From: j mike dupont Date: Mon, 15 Apr 2024 20:05:36 +0000 Subject: [PATCH 04/28] infrastructure pip freeze github actions and docker compose update update run update adding steps working as root run update update Add ARM Dockerfile Temporarily require peft<0.5.0, transformers<4.32.0 (#470) Peft 0.5 recently released and broke some compatilibities. This PR temporarily requires petals to use the previous stable version of peft while we work on 0.5.0 support. run update adding steps working as root petals inference moving to my org first docker compose move to our org format adding hive mind back into the setup reformat ipynb now to test main version using cpu now working. The health server now needs to wait for the others to come up but otherwise it is working. adding chat health adding new test run update Update run-tests-docker.yaml tpu starting running locally in cpu mode, now we have the basic directory structure for an env, still need to tag the items properly. versions building locally, same problem new env pip now loading update inference adding path .env now the .env file is passed in explicitly. This must be wrong somewhere. usage like this `sudo docker compose --profile core --env-file ./.env up` --- .github/workflows/check-style.yaml | 11 ++- .github/workflows/push-docker-image.yaml | 14 +-- .github/workflows/run-tests-docker.yaml | 36 +++++++ .github/workflows/run-tests.yaml | 8 +- Dockerfile | 14 ++- README.md | 22 +++++ docker-compose.yml | 116 +++++++++++++++++++++++ envs/cpu/is1/env.txt | 2 + envs/gpu/h100.txt | 2 + envs/gpu/h100/peers.txt | 1 + envs/gpu/is1/env.txt | 5 + envs/tpu/v3-0/env.txt | 4 + etc/petals-inference.service | 11 +++ get_peersl.sh | 1 + health.env | 3 + pip.freeze | 70 ++++++++++++++ pip.freeze2 | 3 + pyproject.toml | 2 +- run.sh | 1 + run2.sh | 1 + src/petals/__init__.py | 8 +- up.sh | 1 + 22 files changed, 312 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/run-tests-docker.yaml create mode 100644 docker-compose.yml create mode 100644 envs/cpu/is1/env.txt create mode 100644 envs/gpu/h100.txt create mode 100644 envs/gpu/h100/peers.txt create mode 100644 envs/gpu/is1/env.txt create mode 100644 envs/tpu/v3-0/env.txt create mode 100644 etc/petals-inference.service create mode 100644 get_peersl.sh create mode 100644 health.env create mode 100644 pip.freeze create mode 100644 pip.freeze2 create mode 100755 run.sh create mode 100644 run2.sh create mode 100644 up.sh diff --git a/.github/workflows/check-style.yaml b/.github/workflows/check-style.yaml index 60ea42b42..d878766d0 100644 --- a/.github/workflows/check-style.yaml +++ b/.github/workflows/check-style.yaml @@ -9,18 +9,19 @@ jobs: black: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: psf/black@stable + - uses: meta-introspector/checkout@main + - uses: meta-introspector/black@main with: options: "--check --diff" version: "22.3.0" isort: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: meta-introspector/checkout@main + - uses: meta-introspector/setup-python@main with: python-version: 3.8 - - uses: isort/isort-action@master + - uses: meta-introspector/isort-action@main with: isortVersion: "5.10.1" + diff --git a/.github/workflows/push-docker-image.yaml b/.github/workflows/push-docker-image.yaml index 58fbb0030..88ae1294d 100644 --- a/.github/workflows/push-docker-image.yaml +++ b/.github/workflows/push-docker-image.yaml @@ -14,15 +14,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: meta-introspector/checkout@main - name: Docker meta id: meta - uses: crazy-max/ghaction-docker-meta@v2 + uses: meta-introspector/metadata-action@main with: # list of Docker images to use as base name for tags images: | - learningathome/petals + h4ckermike/petals # generate Docker tags based on the following events/attributes tags: | type=ref,event=branch @@ -33,17 +33,17 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: meta-introspector/setup-buildx-action@main - name: Login to Docker Hub if: github.event_name != 'pull_request' - uses: docker/login-action@v1 + uses: meta-introspector/login-action@main with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Free disk space on Ubuntu runner - uses: kfir4444/free-disk-space@main + uses: meta-introspector/free-disk-space@main with: # found in: https://github.com/docker/build-push-action/issues/968 tool-cache: false @@ -55,7 +55,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: meta-introspector/build-push-action@main with: context: . push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/run-tests-docker.yaml b/.github/workflows/run-tests-docker.yaml new file mode 100644 index 000000000..6f6f00f18 --- /dev/null +++ b/.github/workflows/run-tests-docker.yaml @@ -0,0 +1,36 @@ +name: Tests in docker compose + +on: + push: + branches: [ main ] + pull_request: + +jobs: + run-tests-in-compose: + # runs-on: ubuntu-latest + runs-on: self-hosted + timeout-minutes: 20 + steps: + - name: Increase swap space + if: ${{ matrix.os == 'ubuntu' }} + uses: meta-introspector/set-swap-space@main + with: + swap-size-gb: 10 + - name: Checkout + uses: meta-introspector/checkout@main + + - name: Build the docker-compose stack + run: docker-compose -f docker-compose.yml up -d + + - name: Check running containers + run: docker ps -a + + - name: Check logs + run: docker logs health + + - name: Build the docker-compose stack + run: docker-compose down + + + + diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d6316d48b..2ccc72951 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -24,17 +24,17 @@ jobs: steps: - name: Increase swap space if: ${{ matrix.os == 'ubuntu' }} - uses: pierotofy/set-swap-space@master + uses: meta-introspector/set-swap-space@main with: swap-size-gb: 10 - name: Checkout - uses: actions/checkout@v3 + uses: meta-introspector/checkout@main - name: Set up Python - uses: actions/setup-python@v3 + uses: meta-introspector/setup-python@main with: python-version: ${{ matrix.python-version }} - name: Cache dependencies - uses: actions/cache@v3 + uses: meta-introspector/cache@main with: path: ~/.cache/pip key: Key-v1-${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} diff --git a/Dockerfile b/Dockerfile index b1a267674..661cc7737 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM nvcr.io/nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 -LABEL maintainer="bigscience-workshop" +LABEL maintainer="meta-introspector" LABEL repository="petals" WORKDIR /home @@ -23,9 +23,17 @@ RUN conda install python~=3.10.12 pip && \ VOLUME /cache ENV PETALS_CACHE=/cache +COPY pip.freeze petals/pip.freeze +RUN pip install --no-cache-dir -r petals/pip.freeze +COPY pip.freeze2 petals/pip.freeze2 +RUN pip install --no-cache-dir -r petals/pip.freeze2 -COPY . petals/ + +ADD tests petals/tests +ADD LICENSE README.md pyproject.toml setup.cfg petals/ +ADD src petals/src RUN pip install --no-cache-dir -e petals +RUN pip install --no-cache-dir --upgrade transformers==4.34.0 WORKDIR /home/petals/ -CMD bash +CMD python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 \ No newline at end of file diff --git a/README.md b/README.md index 63449ae11..d880286c4 100644 --- a/README.md +++ b/README.md @@ -144,3 +144,25 @@ _arXiv preprint arXiv:2209.01188,_ 2022.

+ + +# setup + + + 1623 sudo cp petals-inference.service /etc/systemd/system/ + 1634 sudo systemctl daemon-reload + 1635 sudo systemctl status petals-inference.service -l + 1636 sudo systemctl restart petals-inference.service -l + + 1639 sudo useradd petals + 1640 sudo mkdir /home/petals + 1641 sudo chown petals: /home/petals/ + 1643 sudo cp -r ~/.venv/ /home/petals/venv + 1644 sudo rm -rf /home/petals/venv + 1658 sudo mv ~/.venv/ /home/petals/venv + 1659 sudo chown petals: /home/petals/ + +1670 sudo systemctl status petals-inference.service -l + 1674 sudo systemctl restart petals-inference.service -l + 1675 sudo systemctl status petals-inference.service -l + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..4a2b33f8b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,116 @@ +services: + backbone: + profiles: ["core"] + image: h4ckermike/petals:main + command: python -m petals.cli.run_dht --host_maddrs /ip4/0.0.0.0/tcp/8008 --identity_path /cache/bootstrap1.id + volumes: + - petals-cache-backbone:/cache + network_mode: host + ipc: host + restart: unless-stopped + env_file: ./.env + + health: + profiles: ["core"] + restart: always + depends_on: + - backbone + image: h4ckermike/health.petals:main + ports: + - "8009:5000" + command: flask run --host=0.0.0.0 --port=5000 + env_file: ./.env + + tinyllama_local_gpu: + profiles: ["local","gpu"] + # image: h4ckermike/petals:main + build : . + depends_on: + - backbone + ports: + - "31336:31336" + command: python -m petals.cli.run_server --port 31336 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: always + env_file: ./.env + + tinyllama_local_cpu: + profiles: ["local","cpu"] + build: . + depends_on: + - backbone + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE + ports: + - "31331:31331" + restart: always + env_file: ./.env + + tinyllamacpu: + profiles: ["tinyllama","cpu"] + image: h4ckermike/petals:main + depends_on: + - backbone + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE + ports: + - "31331:31331" + restart: always + env_file: ./.env + + tinyllamagpu: + profiles: ["core"] + image: h4ckermike/petals:main + depends_on: + - backbone + ports: + - "31332:31332" + command: python -m petals.cli.run_server --port 31332 --num_blocks=100 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + restart: always + env_file: ./.env + + tinyllamatpu: + profiles: ["tpu"] + image: h4ckermike/petals:main + depends_on: + - backbone + ports: + - "31333:31333" + command: python -m petals.cli.run_server --port 31333 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE + restart: always + env_file: ./.env + + debug_health: + profiles: ["debug"] + + image: h4ckermike/health.petals:main + command: bash + stdin_open: true + tty: true + + inference : + profiles: ["core"] + restart: always + depends_on: + - backbone +# image: h4ckermike/inference.petals:main + image: petals-inference-test + ports: + - "8010:5000" + env_file: ./.env + command: gunicorn app:app --bind 0.0.0.0:5000 --worker-class gthread --threads 100 --timeout 1000 + + +volumes: + petals-cache-backbone: diff --git a/envs/cpu/is1/env.txt b/envs/cpu/is1/env.txt new file mode 100644 index 000000000..67a88f2cb --- /dev/null +++ b/envs/cpu/is1/env.txt @@ -0,0 +1,2 @@ +INITIAL_PEERS=/ip4/172.17.0.1/tcp/8099/p2p/QmfVvYv3w3EqpKGYG5FCcER9bFgoGLCUvXDUJsZAgSDw3N +DEVICE=cpu diff --git a/envs/gpu/h100.txt b/envs/gpu/h100.txt new file mode 100644 index 000000000..dce85093d --- /dev/null +++ b/envs/gpu/h100.txt @@ -0,0 +1,2 @@ +INITIAL_PEERS=/ip4/216.81.245.26/tcp/8008/p2p/QmR4PcZvHg414Q2HNEaQZLiu69HD4Vs17hcwDgq8qJdJq2 +DEVICE=cuda diff --git a/envs/gpu/h100/peers.txt b/envs/gpu/h100/peers.txt new file mode 100644 index 000000000..6733d8404 --- /dev/null +++ b/envs/gpu/h100/peers.txt @@ -0,0 +1 @@ +/ip4/216.81.245.26/tcp/8099/p2p/QmR4PcZvHg414Q2HNEaQZLiu69HD4Vs17hcwDgq8qJdJq2 diff --git a/envs/gpu/is1/env.txt b/envs/gpu/is1/env.txt new file mode 100644 index 000000000..5cfce73ed --- /dev/null +++ b/envs/gpu/is1/env.txt @@ -0,0 +1,5 @@ +INITIAL_PEERS=/ip4/172.17.0.1/tcp/8099/p2p/QmfVvYv3w3EqpKGYG5FCcER9bFgoGLCUvXDUJsZAgSDw3N +#PJRT_DEVICE=TPU +DEVICE=cuda +#DEVICE=cpu +#DEVICE=tpux diff --git a/envs/tpu/v3-0/env.txt b/envs/tpu/v3-0/env.txt new file mode 100644 index 000000000..c0c3c0768 --- /dev/null +++ b/envs/tpu/v3-0/env.txt @@ -0,0 +1,4 @@ +INITIAL_PEERS=/ip4/10.164.0.22/tcp/8099/p2p/QmRVmvteSpVKKeNDSaV7Ezy3HNA4bnNfE2EbzDJVFDEwAa +#PJRT_DEVICE=TPU +#DEVICE=xla +DEVICE=cpu \ No newline at end of file diff --git a/etc/petals-inference.service b/etc/petals-inference.service new file mode 100644 index 000000000..ae08b6f40 --- /dev/null +++ b/etc/petals-inference.service @@ -0,0 +1,11 @@ +[Unit] +Description=Petals Inference + +[Service] +User=petals +Group=petals +Environment=PJRT_DEVICE=TPU +ExecStart=/home/petals/venv/bin/python -m petals.cli.run_server --port 31330 petals-team/StableBeluga2 --device xla --num_blocks=4 + +[Install] +WantedBy=multi-user.target diff --git a/get_peersl.sh b/get_peersl.sh new file mode 100644 index 000000000..5dcd3abaa --- /dev/null +++ b/get_peersl.sh @@ -0,0 +1 @@ +docker logs petals-backbone-1 2>&1 |grep initial_peers |cut "-d " -f18- | sort -u > peers.txt diff --git a/health.env b/health.env new file mode 100644 index 000000000..20e15a404 --- /dev/null +++ b/health.env @@ -0,0 +1,3 @@ +INITIAL_PEERS=/ip4/10.164.0.22/tcp/8099/p2p/QmRVmvteSpVKKeNDSaV7Ezy3HNA4bnNfE2EbzDJVFDEwAa +PJRT_DEVICE=TPU +DEVICE=xla \ No newline at end of file diff --git a/pip.freeze b/pip.freeze new file mode 100644 index 000000000..43d0b4ada --- /dev/null +++ b/pip.freeze @@ -0,0 +1,70 @@ +accelerate==0.29.2 +async-timeout==4.0.3 +base58==2.1.1 +bitsandbytes==0.41.1 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +ConfigArgParse==1.7 +cpufeature==0.2.1 +cryptography==42.0.5 +Dijkstar==2.6.0 +filelock==3.13.4 +fsspec==2024.3.1 +grpcio==1.62.1 +grpcio-tools==1.62.1 +hivemind==1.1.10.post2 +#huggingface-hub==0.22.2 +humanfriendly==10.0 +idna==3.7 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +mpmath==1.3.0 +msgpack==1.0.8 +multiaddr==0.0.9 +netaddr==1.2.1 +networkx==3.3 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.19.3 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.1.105 +packaging==24.0 +peft==0.5.0 +prefetch-generator==1.0.3 +protobuf==4.25.3 +psutil==5.9.8 +pycparser==2.22 +pydantic==1.10.15 +pymultihash==0.8.2 +PyYAML==6.0.1 +regex==2023.12.25 +requests==2.31.0 +safetensors==0.4.3 +scipy==1.13.0 +sentencepiece==0.2.0 +six==1.16.0 +sortedcontainers==2.4.0 +speedtest-cli==2.1.3 +sympy==1.12 +tensor-parallel==1.0.23 + +torch==2.2.2 +tqdm==4.66.2 +#transformers==4.38.2 + +triton==2.2.0 +typing_extensions==4.11.0 +urllib3==2.2.1 +uvloop==0.19.0 +varint==1.0.2 + + diff --git a/pip.freeze2 b/pip.freeze2 new file mode 100644 index 000000000..4af24038d --- /dev/null +++ b/pip.freeze2 @@ -0,0 +1,3 @@ +transformers==4.34.0 +tokenizers>=0.14,<0.15 +huggingface_hub>=0.16.4,<0.18 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cfc991c07..6f1475b01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 120 -required-version = "22.3.0" +required-version = "24.3.0" [tool.isort] profile = "black" diff --git a/run.sh b/run.sh new file mode 100755 index 000000000..15bff4390 --- /dev/null +++ b/run.sh @@ -0,0 +1 @@ +PJRT_DEVICE=TPU python -m petals.cli.run_server --port 31330 petals-team/StableBeluga2 --device xla --num_blocks=10 diff --git a/run2.sh b/run2.sh new file mode 100644 index 000000000..23c984d57 --- /dev/null +++ b/run2.sh @@ -0,0 +1 @@ +docker run -d --net host --ipc host --volume petals-cache-backbone:/cache --name backbone --rm learningathome/petals:main python -m petals.cli.run_dht --host_maddrs /ip4/0.0.0.0/tcp/8099 --identity_path bootstrap1.id diff --git a/src/petals/__init__.py b/src/petals/__init__.py index ccc560e0c..52e5af110 100644 --- a/src/petals/__init__.py +++ b/src/petals/__init__.py @@ -20,10 +20,10 @@ __version__ = "2.3.0.dev2" -if not os.getenv("PETALS_IGNORE_DEPENDENCY_VERSION"): - assert ( - version.parse("4.38.2") <= version.parse(transformers.__version__) < version.parse("4.39.0") - ), "Please install a proper transformers version: pip install transformers>=4.37.1,<4.39.0" +#if not os.getenv("PETALS_IGNORE_DEPENDENCY_VERSION"): +# assert ( +# version.parse("4.38.2") <= version.parse(transformers.__version__) < version.parse("4.39.0") +# ), "Please install a proper transformers version: pip install transformers>=4.37.1,<4.39.0" def _override_bfloat16_mode_default(): diff --git a/up.sh b/up.sh new file mode 100644 index 000000000..8ed68d13e --- /dev/null +++ b/up.sh @@ -0,0 +1 @@ +sudo docker compose --profile core --env-file ./envs/gpu/h100.txt up health From 661545f0ff3c91b4a6dc54631fc7efe1ebd7c3df Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 16 Apr 2024 14:05:02 -0400 Subject: [PATCH 05/28] tiny mixtral running locally --- Dockerfile | 10 +++++----- docker-compose.yml | 25 ++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 661cc7737..ac4d033d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,17 +23,17 @@ RUN conda install python~=3.10.12 pip && \ VOLUME /cache ENV PETALS_CACHE=/cache -COPY pip.freeze petals/pip.freeze -RUN pip install --no-cache-dir -r petals/pip.freeze -COPY pip.freeze2 petals/pip.freeze2 -RUN pip install --no-cache-dir -r petals/pip.freeze2 +#COPY pip.freeze petals/pip.freeze +#RUN pip install --no-cache-dir -r petals/pip.freeze +#COPY pip.freeze2 petals/pip.freeze2 +#RUN pip install --no-cache-dir -r petals/pip.freeze2 ADD tests petals/tests ADD LICENSE README.md pyproject.toml setup.cfg petals/ ADD src petals/src RUN pip install --no-cache-dir -e petals -RUN pip install --no-cache-dir --upgrade transformers==4.34.0 +#RUN pip install --no-cache-dir --upgrade transformers==4.34.0 WORKDIR /home/petals/ CMD python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4a2b33f8b..7c8f293ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,26 @@ services: capabilities: [gpu] restart: always env_file: ./.env - + +# + tinymixtral_local_gpu: + profiles: ["local","gpu"] + build: . + depends_on: + - backbone + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 SanjiWatsuki/TinyMixtral-32x248M --initial_peers $INITIAL_PEERS --device=$DEVICE + ports: + - "31331:31331" + restart: always + env_file: ./.env + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + tinyllama_local_cpu: profiles: ["local","cpu"] build: . @@ -104,8 +123,8 @@ services: restart: always depends_on: - backbone -# image: h4ckermike/inference.petals:main - image: petals-inference-test + image: h4ckermike/inference.petals:main +# image: petals-inference-test ports: - "8010:5000" env_file: ./.env From be69f3117f23bb7df549f164de2065309f6454d7 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Tue, 16 Apr 2024 14:58:27 -0400 Subject: [PATCH 06/28] grok --- Dockerfile | 2 ++ docker-compose.yml | 20 ++++++++++++++++++++ src/petals/utils/auto_config.py | 3 +++ 3 files changed, 25 insertions(+) diff --git a/Dockerfile b/Dockerfile index ac4d033d4..48f164055 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,8 @@ ADD tests petals/tests ADD LICENSE README.md pyproject.toml setup.cfg petals/ ADD src petals/src RUN pip install --no-cache-dir -e petals + +RUN pip freeze > pip.freeze.new #RUN pip install --no-cache-dir --upgrade transformers==4.34.0 WORKDIR /home/petals/ diff --git a/docker-compose.yml b/docker-compose.yml index 7c8f293ff..3f7719f8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,26 @@ services: env_file: ./.env # + grok_local_gpu: + profiles: ["local","gpu"] + build: . + depends_on: + - backbone + # xai-org/grok-1 + # hpcai-tech/grok-1 + # keyfan/grok-1-hf + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 keyfan/grok-1-hf --initial_peers $INITIAL_PEERS --device=$DEVICE + ports: + - "31331:31331" + restart: always + env_file: ./.env + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] tinymixtral_local_gpu: profiles: ["local","gpu"] build: . diff --git a/src/petals/utils/auto_config.py b/src/petals/utils/auto_config.py index 0cec83d87..9b0955329 100644 --- a/src/petals/utils/auto_config.py +++ b/src/petals/utils/auto_config.py @@ -40,6 +40,9 @@ def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike, None], *arg ): kwargs["use_auth_token"] = True + kwargs["trust_remote_code"] = True +#trust_remote_code=True + config = AutoConfig.from_pretrained(model_name_or_path, *args, **kwargs) if config.model_type not in _CLASS_MAPPING: raise ValueError(f"Petals does not support model type {config.model_type}") From c98b5328cbb0b76109ead837b044578a74eabef1 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 17 Apr 2024 12:35:10 -0400 Subject: [PATCH 07/28] docker changes --- Dockerfile | 14 +++++++++----- docker-compose.yml | 37 ++++++++++++++++++++++++++++--------- pip.freeze | 15 +++++++++------ pip2.freeze | 6 ++++++ 4 files changed, 52 insertions(+), 20 deletions(-) create mode 100644 pip2.freeze diff --git a/Dockerfile b/Dockerfile index ac4d033d4..1fe2adb5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,17 +23,21 @@ RUN conda install python~=3.10.12 pip && \ VOLUME /cache ENV PETALS_CACHE=/cache -#COPY pip.freeze petals/pip.freeze -#RUN pip install --no-cache-dir -r petals/pip.freeze -#COPY pip.freeze2 petals/pip.freeze2 -#RUN pip install --no-cache-dir -r petals/pip.freeze2 +ADD pip.freeze petals/pip.freeze +RUN pip install --no-cache-dir -r petals/pip.freeze +ADD pip2.freeze petals/pip2.freeze +RUN pip install --no-cache-dir -r petals/pip2.freeze ADD tests petals/tests -ADD LICENSE README.md pyproject.toml setup.cfg petals/ ADD src petals/src +ADD LICENSE README.md pyproject.toml setup.cfg petals/ + RUN pip install --no-cache-dir -e petals #RUN pip install --no-cache-dir --upgrade transformers==4.34.0 WORKDIR /home/petals/ + +RUN pip freeze > pip.freeze.new + CMD python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7c8f293ff..a36276c60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ services: network_mode: host ipc: host restart: unless-stopped - env_file: ./.env health: profiles: ["core"] @@ -19,7 +18,6 @@ services: ports: - "8009:5000" command: flask run --host=0.0.0.0 --port=5000 - env_file: ./.env tinyllama_local_gpu: profiles: ["local","gpu"] @@ -38,9 +36,30 @@ services: count: 1 capabilities: [gpu] restart: always - env_file: ./.env # + envmodel_local_gpu: + profiles: ["local","gpu"] + build: . + environment: + - MODEL=${MODEL} + depends_on: + - backbone + # xai-org/grok-1 + # hpcai-tech/grok-1 + # keyfan/grok-1-hf + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE + ports: + - "31331:31331" + restart: always + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] +>>>>>>> 79bb253 (docker changes) tinymixtral_local_gpu: profiles: ["local","gpu"] build: . @@ -50,7 +69,7 @@ services: ports: - "31331:31331" restart: always - env_file: ./.env + deploy: resources: reservations: @@ -68,7 +87,7 @@ services: ports: - "31331:31331" restart: always - env_file: ./.env + tinyllamacpu: profiles: ["tinyllama","cpu"] @@ -79,7 +98,7 @@ services: ports: - "31331:31331" restart: always - env_file: ./.env + tinyllamagpu: profiles: ["core"] @@ -97,7 +116,7 @@ services: count: 1 capabilities: [gpu] restart: always - env_file: ./.env + tinyllamatpu: profiles: ["tpu"] @@ -108,7 +127,7 @@ services: - "31333:31333" command: python -m petals.cli.run_server --port 31333 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE restart: always - env_file: ./.env + debug_health: profiles: ["debug"] @@ -127,7 +146,7 @@ services: # image: petals-inference-test ports: - "8010:5000" - env_file: ./.env + command: gunicorn app:app --bind 0.0.0.0:5000 --worker-class gthread --threads 100 --timeout 1000 diff --git a/pip.freeze b/pip.freeze index 43d0b4ada..f18d1c71e 100644 --- a/pip.freeze +++ b/pip.freeze @@ -8,16 +8,17 @@ charset-normalizer==3.3.2 ConfigArgParse==1.7 cpufeature==0.2.1 cryptography==42.0.5 + Dijkstar==2.6.0 filelock==3.13.4 fsspec==2024.3.1 grpcio==1.62.1 grpcio-tools==1.62.1 -hivemind==1.1.10.post2 -#huggingface-hub==0.22.2 humanfriendly==10.0 idna==3.7 Jinja2==3.1.3 +jsonpointer==2.1 + MarkupSafe==2.1.5 mpmath==1.3.0 msgpack==1.0.8 @@ -37,6 +38,7 @@ nvidia-cusparse-cu12==12.1.0.106 nvidia-nccl-cu12==2.19.3 nvidia-nvjitlink-cu12==12.4.127 nvidia-nvtx-cu12==12.1.105 + packaging==24.0 peft==0.5.0 prefetch-generator==1.0.3 @@ -48,6 +50,7 @@ pymultihash==0.8.2 PyYAML==6.0.1 regex==2023.12.25 requests==2.31.0 + safetensors==0.4.3 scipy==1.13.0 sentencepiece==0.2.0 @@ -57,14 +60,14 @@ speedtest-cli==2.1.3 sympy==1.12 tensor-parallel==1.0.23 -torch==2.2.2 tqdm==4.66.2 -#transformers==4.38.2 - triton==2.2.0 typing_extensions==4.11.0 urllib3==2.2.1 uvloop==0.19.0 varint==1.0.2 - +triton==2.2.0 +typing_extensions==4.11.0 +uvloop==0.19.0 +varint==1.0.2 diff --git a/pip2.freeze b/pip2.freeze new file mode 100644 index 000000000..3191ffb81 --- /dev/null +++ b/pip2.freeze @@ -0,0 +1,6 @@ +transformers==4.38.2 +huggingface-hub==0.22.2 +hivemind==1.1.10.post2 +peft==0.5.0 +tokenizers==0.15.2 +torch==2.2.2 From 6124622667305fdfcafef9515ca880f76bf3af21 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 17 Apr 2024 12:46:20 -0400 Subject: [PATCH 08/28] mixtral working --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a36276c60..ce1f28782 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,7 +59,7 @@ services: - driver: nvidia count: 1 capabilities: [gpu] ->>>>>>> 79bb253 (docker changes) + tinymixtral_local_gpu: profiles: ["local","gpu"] build: . From 1644e8938552349e636cea2b9f3a42d6ce0ae3a6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 17 Apr 2024 12:56:48 -0400 Subject: [PATCH 09/28] reformat black --- src/petals/__init__.py | 2 +- src/petals/client/ptune.py | 2 +- src/petals/client/remote_forward_backward.py | 1 + src/petals/client/routing/spending_policy.py | 1 + src/petals/client/sequential_autograd.py | 1 + src/petals/models/bloom/block.py | 1 + src/petals/models/falcon/block.py | 1 + src/petals/models/llama/block.py | 1 + src/petals/server/block_functions.py | 1 + src/petals/server/from_pretrained.py | 1 + src/petals/server/memory_cache.py | 1 + src/petals/utils/auto_config.py | 2 +- src/petals/utils/convert_block.py | 1 + src/petals/utils/dht.py | 1 + 14 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/petals/__init__.py b/src/petals/__init__.py index 52e5af110..68120bfc4 100644 --- a/src/petals/__init__.py +++ b/src/petals/__init__.py @@ -20,7 +20,7 @@ __version__ = "2.3.0.dev2" -#if not os.getenv("PETALS_IGNORE_DEPENDENCY_VERSION"): +# if not os.getenv("PETALS_IGNORE_DEPENDENCY_VERSION"): # assert ( # version.parse("4.38.2") <= version.parse(transformers.__version__) < version.parse("4.39.0") # ), "Please install a proper transformers version: pip install transformers>=4.37.1,<4.39.0" diff --git a/src/petals/client/ptune.py b/src/petals/client/ptune.py index f3995d68a..ecd3543e8 100644 --- a/src/petals/client/ptune.py +++ b/src/petals/client/ptune.py @@ -51,7 +51,7 @@ def get_prompt(self, batch_size): batch_size, self.pre_seq_len, self.config.num_hidden_layers, - self.config.hidden_size + self.config.hidden_size, # TODO: should be num_hidden_layers - 1 ) intermediate_prompts = intermediate_prompts.permute([2, 0, 1, 3]) diff --git a/src/petals/client/remote_forward_backward.py b/src/petals/client/remote_forward_backward.py index 44abe2686..d61106702 100644 --- a/src/petals/client/remote_forward_backward.py +++ b/src/petals/client/remote_forward_backward.py @@ -1,6 +1,7 @@ """ Utility functions that call RPC forward or backward on a single remote server """ + import asyncio from typing import Iterable, List, Optional, Sequence, Tuple diff --git a/src/petals/client/routing/spending_policy.py b/src/petals/client/routing/spending_policy.py index 0af3db7fd..f4eddba76 100644 --- a/src/petals/client/routing/spending_policy.py +++ b/src/petals/client/routing/spending_policy.py @@ -3,6 +3,7 @@ The intent is to let Petals participants earn points by helping others while idle (e.g. at night), then use these points to run their own compute experiments faster. See Section 4 of https://arxiv.org/abs/2209.01188 for discussion. """ + from abc import ABC, abstractmethod diff --git a/src/petals/client/sequential_autograd.py b/src/petals/client/sequential_autograd.py index 9d965d2a5..c95f0a701 100644 --- a/src/petals/client/sequential_autograd.py +++ b/src/petals/client/sequential_autograd.py @@ -1,6 +1,7 @@ """ A PyTorch autograd function that runs forward/backward on a sequence of remote servers in a fault-tolerant manner """ + import asyncio import itertools from collections import deque diff --git a/src/petals/models/bloom/block.py b/src/petals/models/bloom/block.py index 439b9ca10..d7431860c 100644 --- a/src/petals/models/bloom/block.py +++ b/src/petals/models/bloom/block.py @@ -3,6 +3,7 @@ Based on https://github.com/huggingface/transformers/commit/ca2a55e9dfb245527b5e1c954fec6ffbb7aef07b See commit history for authorship. """ + from typing import Optional, Tuple import torch diff --git a/src/petals/models/falcon/block.py b/src/petals/models/falcon/block.py index a510abaa1..761bd5dd1 100644 --- a/src/petals/models/falcon/block.py +++ b/src/petals/models/falcon/block.py @@ -3,6 +3,7 @@ Based on https://github.com/huggingface/transformers/blob/main/src/transformers/models/falcon/modeling_falcon.py See commit history for authorship. """ + import math from functools import partial from typing import Optional, Tuple diff --git a/src/petals/models/llama/block.py b/src/petals/models/llama/block.py index 2eb8f731f..bd6c8c86d 100644 --- a/src/petals/models/llama/block.py +++ b/src/petals/models/llama/block.py @@ -3,6 +3,7 @@ Based on https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py See commit history for authorship. """ + import math from typing import Optional, Tuple diff --git a/src/petals/server/block_functions.py b/src/petals/server/block_functions.py index a79f05c94..4c3cf9a02 100644 --- a/src/petals/server/block_functions.py +++ b/src/petals/server/block_functions.py @@ -1,6 +1,7 @@ """ This module implements server-side computations on served blocks: forward, backward and inference; used by handler """ + from __future__ import annotations from typing import Any, AsyncIterator, Dict, Optional, Sequence, Tuple, Union diff --git a/src/petals/server/from_pretrained.py b/src/petals/server/from_pretrained.py index 4a3b15077..bad0b2ef0 100644 --- a/src/petals/server/from_pretrained.py +++ b/src/petals/server/from_pretrained.py @@ -6,6 +6,7 @@ - fetch the weights over IPoAC, using a fleet of trained pigeons ( http://www.faqs.org/rfcs/rfc1149.html ) """ + import json import time from contextlib import suppress diff --git a/src/petals/server/memory_cache.py b/src/petals/server/memory_cache.py index fa4db218f..9e7abf7b2 100644 --- a/src/petals/server/memory_cache.py +++ b/src/petals/server/memory_cache.py @@ -4,6 +4,7 @@ For now, the only purpose of this code is to ensure that allocated memory will be deleted properly. """ + import asyncio import contextlib import ctypes diff --git a/src/petals/utils/auto_config.py b/src/petals/utils/auto_config.py index 9b0955329..6043c7ba2 100644 --- a/src/petals/utils/auto_config.py +++ b/src/petals/utils/auto_config.py @@ -41,7 +41,7 @@ def from_pretrained(cls, model_name_or_path: Union[str, os.PathLike, None], *arg kwargs["use_auth_token"] = True kwargs["trust_remote_code"] = True -#trust_remote_code=True + # trust_remote_code=True config = AutoConfig.from_pretrained(model_name_or_path, *args, **kwargs) if config.model_type not in _CLASS_MAPPING: diff --git a/src/petals/utils/convert_block.py b/src/petals/utils/convert_block.py index 94d3e29f3..26d6b7dd3 100644 --- a/src/petals/utils/convert_block.py +++ b/src/petals/utils/convert_block.py @@ -1,6 +1,7 @@ """ Tools for converting transformer blocks, applying quantization and/or tensor parallelism """ + import re from enum import Enum from typing import Optional, Sequence diff --git a/src/petals/utils/dht.py b/src/petals/utils/dht.py index 4faf74aa7..357cd98cc 100644 --- a/src/petals/utils/dht.py +++ b/src/petals/utils/dht.py @@ -1,6 +1,7 @@ """ Utilities for declaring and retrieving active model layers using a shared DHT. """ + from __future__ import annotations import math From db5cb42d4063e8f383bbba912cfab8a396f5adde Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 29 Apr 2024 14:21:31 -0400 Subject: [PATCH 10/28] update config --- src/petals/constants.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/petals/constants.py b/src/petals/constants.py index d307b8140..3425ea6b8 100644 --- a/src/petals/constants.py +++ b/src/petals/constants.py @@ -2,17 +2,10 @@ PUBLIC_INITIAL_PEERS = [ # IPv4 DNS addresses - "/dns/bootstrap1.petals.dev/tcp/31337/p2p/QmedTaZXmULqwspJXz44SsPZyTNKxhnnFvYRajfH7MGhCY", - "/dns/bootstrap2.petals.dev/tcp/31338/p2p/QmQGTqmM7NKjV6ggU1ZCap8zWiyKR89RViDXiqehSiCpY5", - # IPv6 DNS addresses - "/dns6/bootstrap1.petals.dev/tcp/31337/p2p/QmedTaZXmULqwspJXz44SsPZyTNKxhnnFvYRajfH7MGhCY", - "/dns6/bootstrap2.petals.dev/tcp/31338/p2p/QmQGTqmM7NKjV6ggU1ZCap8zWiyKR89RViDXiqehSiCpY5", - # Reserved IPs - "/ip4/159.89.214.152/tcp/31337/p2p/QmedTaZXmULqwspJXz44SsPZyTNKxhnnFvYRajfH7MGhCY", - "/ip4/159.203.156.48/tcp/31338/p2p/QmQGTqmM7NKjV6ggU1ZCap8zWiyKR89RViDXiqehSiCpY5", + "/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2" ] # The reachability API is currently used only when connecting to the public swarm -REACHABILITY_API_URL = "https://health.petals.dev" +REACHABILITY_API_URL = "https://health.cillium.dev.compute.agentartificial.com" DTYPE_MAP = dict(bfloat16=torch.bfloat16, float16=torch.float16, float32=torch.float32, auto="auto") From f2092acd27027d0977c23f0353d8223ac0daf67a Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 29 Apr 2024 16:08:18 -0400 Subject: [PATCH 11/28] remove deps --- docker-compose.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ce1f28782..0658ff3d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,8 @@ services: health: profiles: ["core"] restart: always - depends_on: - - backbone +# depends_on: +# - backbone image: h4ckermike/health.petals:main ports: - "8009:5000" @@ -23,8 +23,8 @@ services: profiles: ["local","gpu"] # image: h4ckermike/petals:main build : . - depends_on: - - backbone +# depends_on: +# - backbone ports: - "31336:31336" command: python -m petals.cli.run_server --port 31336 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE @@ -43,8 +43,8 @@ services: build: . environment: - MODEL=${MODEL} - depends_on: - - backbone +# depends_on: +# - backbone # xai-org/grok-1 # hpcai-tech/grok-1 # keyfan/grok-1-hf @@ -63,8 +63,8 @@ services: tinymixtral_local_gpu: profiles: ["local","gpu"] build: . - depends_on: - - backbone +# depends_on: +# - backbone command: python -m petals.cli.run_server --port 31331 --num_blocks=1 SanjiWatsuki/TinyMixtral-32x248M --initial_peers $INITIAL_PEERS --device=$DEVICE ports: - "31331:31331" @@ -81,8 +81,8 @@ services: tinyllama_local_cpu: profiles: ["local","cpu"] build: . - depends_on: - - backbone +# depends_on: +# - backbone command: python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE ports: - "31331:31331" @@ -103,8 +103,8 @@ services: tinyllamagpu: profiles: ["core"] image: h4ckermike/petals:main - depends_on: - - backbone +# depends_on: +# - backbone ports: - "31332:31332" command: python -m petals.cli.run_server --port 31332 --num_blocks=100 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE @@ -121,8 +121,8 @@ services: tinyllamatpu: profiles: ["tpu"] image: h4ckermike/petals:main - depends_on: - - backbone +# depends_on: +# - backbone ports: - "31333:31333" command: python -m petals.cli.run_server --port 31333 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE @@ -140,8 +140,8 @@ services: inference : profiles: ["core"] restart: always - depends_on: - - backbone +# depends_on: +# - backbone image: h4ckermike/inference.petals:main # image: petals-inference-test ports: From 57442053571c233e6528fc00f48ec6019e67c477 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 29 Apr 2024 20:11:40 +0000 Subject: [PATCH 12/28] added setup.sh and added .env.example --- .env.example | 3 + docker-compose.yml | 126 +--------------------------------------- envs/gpu/h100/peers.txt | 2 +- get_peersl.sh | 2 +- setup.sh | 19 ++++++ 5 files changed, 25 insertions(+), 127 deletions(-) create mode 100644 .env.example create mode 100644 setup.sh diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..f4e0e5128 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +INITIAL_PEERS= +DEVICE=CUDA +MODEL=meta-llama/Meta-Llama-8B-Instruct diff --git a/docker-compose.yml b/docker-compose.yml index ce1f28782..ce93015a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,53 +1,19 @@ services: backbone: profiles: ["core"] - image: h4ckermike/petals:main + image: h4ckermike/petals:main command: python -m petals.cli.run_dht --host_maddrs /ip4/0.0.0.0/tcp/8008 --identity_path /cache/bootstrap1.id volumes: - petals-cache-backbone:/cache network_mode: host ipc: host restart: unless-stopped - - health: - profiles: ["core"] - restart: always - depends_on: - - backbone - image: h4ckermike/health.petals:main - ports: - - "8009:5000" - command: flask run --host=0.0.0.0 --port=5000 - tinyllama_local_gpu: - profiles: ["local","gpu"] - # image: h4ckermike/petals:main - build : . - depends_on: - - backbone - ports: - - "31336:31336" - command: python -m petals.cli.run_server --port 31336 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu] - restart: always - -# envmodel_local_gpu: profiles: ["local","gpu"] build: . environment: - MODEL=${MODEL} - depends_on: - - backbone - # xai-org/grok-1 - # hpcai-tech/grok-1 - # keyfan/grok-1-hf command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE ports: - "31331:31331" @@ -60,95 +26,5 @@ services: count: 1 capabilities: [gpu] - tinymixtral_local_gpu: - profiles: ["local","gpu"] - build: . - depends_on: - - backbone - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 SanjiWatsuki/TinyMixtral-32x248M --initial_peers $INITIAL_PEERS --device=$DEVICE - ports: - - "31331:31331" - restart: always - - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu] - - tinyllama_local_cpu: - profiles: ["local","cpu"] - build: . - depends_on: - - backbone - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE - ports: - - "31331:31331" - restart: always - - - tinyllamacpu: - profiles: ["tinyllama","cpu"] - image: h4ckermike/petals:main - depends_on: - - backbone - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE - ports: - - "31331:31331" - restart: always - - - tinyllamagpu: - profiles: ["core"] - image: h4ckermike/petals:main - depends_on: - - backbone - ports: - - "31332:31332" - command: python -m petals.cli.run_server --port 31332 --num_blocks=100 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: 1 - capabilities: [gpu] - restart: always - - - tinyllamatpu: - profiles: ["tpu"] - image: h4ckermike/petals:main - depends_on: - - backbone - ports: - - "31333:31333" - command: python -m petals.cli.run_server --port 31333 --num_blocks=1 Maykeye/TinyLLama-v0 --initial_peers $INITIAL_PEERS --device=$DEVICE - restart: always - - - debug_health: - profiles: ["debug"] - - image: h4ckermike/health.petals:main - command: bash - stdin_open: true - tty: true - - inference : - profiles: ["core"] - restart: always - depends_on: - - backbone - image: h4ckermike/inference.petals:main -# image: petals-inference-test - ports: - - "8010:5000" - - command: gunicorn app:app --bind 0.0.0.0:5000 --worker-class gthread --threads 100 --timeout 1000 - - volumes: petals-cache-backbone: diff --git a/envs/gpu/h100/peers.txt b/envs/gpu/h100/peers.txt index 6733d8404..fa750b647 100644 --- a/envs/gpu/h100/peers.txt +++ b/envs/gpu/h100/peers.txt @@ -1 +1 @@ -/ip4/216.81.245.26/tcp/8099/p2p/QmR4PcZvHg414Q2HNEaQZLiu69HD4Vs17hcwDgq8qJdJq2 +INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 diff --git a/get_peersl.sh b/get_peersl.sh index 5dcd3abaa..b14ee899a 100644 --- a/get_peersl.sh +++ b/get_peersl.sh @@ -1 +1 @@ -docker logs petals-backbone-1 2>&1 |grep initial_peers |cut "-d " -f18- | sort -u > peers.txt +docker logs petals-backbone-1 2>&1 | grep initial_peers | cut "-d " -f18- | sort -u > peers.txt diff --git a/setup.sh b/setup.sh new file mode 100644 index 000000000..d72188c49 --- /dev/null +++ b/setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +if [ -z "$INITIAL_PEERS" ]; then + INITIAL_PEERS=cat envs/gpu/h100/peers.txt +fi + +cat env.example > .env +sed -i "s/INITIAL_PEERS=.*/INITIAL_PEERS=$INITIAL_PEERS/" .env + +python -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip + +pip install setuptools wheel gnureadline +pip install -e . + +docker compose --profile core --env-file .env up -d \ No newline at end of file From 1fcc81bccf18d298a2729211094b9cd6d53c5715 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 29 Apr 2024 16:40:04 -0400 Subject: [PATCH 13/28] update --- docker-compose.yml | 9 +++++---- ...1.cillium.dev.compute.agentartificial.com.txt | 4 ++++ setup.sh | 16 ++-------------- 3 files changed, 11 insertions(+), 18 deletions(-) create mode 100644 envs/dht1.cillium.dev.compute.agentartificial.com.txt diff --git a/docker-compose.yml b/docker-compose.yml index ce71c5969..a7b7af9a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: backbone: - profiles: ["core"] + profiles: ["backbone"] image: h4ckermike/petals:main command: python -m petals.cli.run_dht --host_maddrs /ip4/0.0.0.0/tcp/8008 --identity_path /cache/bootstrap1.id volumes: @@ -8,9 +8,10 @@ services: network_mode: host ipc: host restart: unless-stopped - envmodel_local_gpu: - profiles: ["local","gpu"] - build: . + envmodel_gpu: + profiles: ["miner","gpu"] +# build: . + image: h4ckermike/petals:main environment: - MODEL=${MODEL} - INITIAL_PEERS=${INITIAL_PEERS} diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt new file mode 100644 index 000000000..200a5abe4 --- /dev/null +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -0,0 +1,4 @@ +INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 +DEVICE=cuda +MODEL=Maykeye/TinyLLama-v0 +#MODEL=SanjiWatsuki/TinyMixtral-32x248M diff --git a/setup.sh b/setup.sh index d72188c49..e1afef6c8 100644 --- a/setup.sh +++ b/setup.sh @@ -2,18 +2,6 @@ set -e -if [ -z "$INITIAL_PEERS" ]; then - INITIAL_PEERS=cat envs/gpu/h100/peers.txt -fi +cp envs/gpu/dht1.cillium.dev.compute.agentartificial.com.txt .env -cat env.example > .env -sed -i "s/INITIAL_PEERS=.*/INITIAL_PEERS=$INITIAL_PEERS/" .env - -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip - -pip install setuptools wheel gnureadline -pip install -e . - -docker compose --profile core --env-file .env up -d \ No newline at end of file +docker compose --profile miner --env-file .env up -d From 0bfd0cbd48ddce9177159b3647e39d57e9ae7b3d Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 29 Apr 2024 16:41:08 -0400 Subject: [PATCH 14/28] updat --- setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index e1afef6c8..5608bf23c 100644 --- a/setup.sh +++ b/setup.sh @@ -2,6 +2,5 @@ set -e -cp envs/gpu/dht1.cillium.dev.compute.agentartificial.com.txt .env - +cp envs/dht1.cillium.dev.compute.agentartificial.com.txt .env docker compose --profile miner --env-file .env up -d From d9ba0ef0c196a0dcb793eaa0108f231cd18ed7ee Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 29 Apr 2024 16:47:03 -0400 Subject: [PATCH 15/28] onlineer --- setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index 5608bf23c..990601647 100644 --- a/setup.sh +++ b/setup.sh @@ -2,5 +2,4 @@ set -e -cp envs/dht1.cillium.dev.compute.agentartificial.com.txt .env -docker compose --profile miner --env-file .env up -d +docker compose --profile miner --env-file envs/dht1.cillium.dev.compute.agentartificial.com.txt up -d From b84507c15e1e809d69bc0d7277b61c3e558f0bfe Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 2 May 2024 16:11:11 -0400 Subject: [PATCH 16/28] changing defaults to set disk space --- docker-compose.yml | 15 ++++----------- ...t1.cillium.dev.compute.agentartificial.com.txt | 2 ++ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a7b7af9a3..bc5af77b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,15 @@ services: - backbone: - profiles: ["backbone"] - image: h4ckermike/petals:main - command: python -m petals.cli.run_dht --host_maddrs /ip4/0.0.0.0/tcp/8008 --identity_path /cache/bootstrap1.id - volumes: - - petals-cache-backbone:/cache - network_mode: host - ipc: host - restart: unless-stopped envmodel_gpu: profiles: ["miner","gpu"] # build: . image: h4ckermike/petals:main environment: + - MAX_DISK_SPACE=${MAX_DISK_SPACE} + - PUBLIC_NAME=${PUBLIC_NAME} - MODEL=${MODEL} - INITIAL_PEERS=${INITIAL_PEERS} - DEVICE=${DEVICE} - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public-name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE ports: - "31331:31331" restart: always @@ -25,7 +18,7 @@ services: reservations: devices: - driver: nvidia - count: 1 + count: all capabilities: [gpu] volumes: petals-cache-backbone: diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index 200a5abe4..43d0dee4f 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -2,3 +2,5 @@ INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb DEVICE=cuda MODEL=Maykeye/TinyLLama-v0 #MODEL=SanjiWatsuki/TinyMixtral-32x248M +PUBLIC_NAME=NameNotConfigured +MAX_DISK_SPACE=30GB From 804de46b2236f52c5856efaf8863180eafec2e85 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 8 May 2024 15:34:42 -0400 Subject: [PATCH 17/28] sauerkraut --- envs/dht1.cillium.dev.compute.agentartificial.com.txt | 4 ++-- envs/dht1.cillium.prod.compute.agentartificial.com.txt | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 envs/dht1.cillium.prod.compute.agentartificial.com.txt diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index 43d0dee4f..86ec32139 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -1,6 +1,6 @@ INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 DEVICE=cuda -MODEL=Maykeye/TinyLLama-v0 -#MODEL=SanjiWatsuki/TinyMixtral-32x248M +#MODEL=Maykeye/TinyLLama-v0 +MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B PUBLIC_NAME=NameNotConfigured MAX_DISK_SPACE=30GB diff --git a/envs/dht1.cillium.prod.compute.agentartificial.com.txt b/envs/dht1.cillium.prod.compute.agentartificial.com.txt new file mode 100644 index 000000000..86ec32139 --- /dev/null +++ b/envs/dht1.cillium.prod.compute.agentartificial.com.txt @@ -0,0 +1,6 @@ +INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 +DEVICE=cuda +#MODEL=Maykeye/TinyLLama-v0 +MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B +PUBLIC_NAME=NameNotConfigured +MAX_DISK_SPACE=30GB From 41a4aba0ce5208f40da39b638783cb4988141a6f Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 8 May 2024 15:46:55 -0400 Subject: [PATCH 18/28] remove options --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index bc5af77b6..4ca980253 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,9 @@ services: - MODEL=${MODEL} - INITIAL_PEERS=${INITIAL_PEERS} - DEVICE=${DEVICE} - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public-name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE + # --max_disk_space $MAX_DISK_SPACE + # --public-name $PUBLIC_NAME ports: - "31331:31331" restart: always From e59cc855ae093ad74637982e95797b6ee37c5f12 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 8 May 2024 15:53:29 -0400 Subject: [PATCH 19/28] block count --- docker-compose.yml | 3 ++- envs/dht1.cillium.dev.compute.agentartificial.com.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4ca980253..89b11cfbb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,8 @@ services: - MODEL=${MODEL} - INITIAL_PEERS=${INITIAL_PEERS} - DEVICE=${DEVICE} - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE + - BLOCKS=${BLOCKS} + command: python -m petals.cli.run_server --port 31331 --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE # --max_disk_space $MAX_DISK_SPACE # --public-name $PUBLIC_NAME ports: diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index 86ec32139..efbd26b98 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -4,3 +4,4 @@ DEVICE=cuda MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B PUBLIC_NAME=NameNotConfigured MAX_DISK_SPACE=30GB +BLOCKS=30 From 1615f5ed038a76f21f6f4b20493c08798aaa1175 Mon Sep 17 00:00:00 2001 From: Mike DuPont Date: Wed, 8 May 2024 22:14:13 -0400 Subject: [PATCH 20/28] Update dht1.cillium.prod.compute.agentartificial.com.txt --- envs/dht1.cillium.prod.compute.agentartificial.com.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/envs/dht1.cillium.prod.compute.agentartificial.com.txt b/envs/dht1.cillium.prod.compute.agentartificial.com.txt index 86ec32139..6b922cd05 100644 --- a/envs/dht1.cillium.prod.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.prod.compute.agentartificial.com.txt @@ -1,6 +1,7 @@ -INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 +INITIAL_PEERS=/dns/dht1.cillium.prod.compute.agentartificial.com/tcp/8008/p2p/QmRuzeEHPohDR4BGvArrBrqUNC2YPvTc1m5iwTXefdKra8 DEVICE=cuda #MODEL=Maykeye/TinyLLama-v0 MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B PUBLIC_NAME=NameNotConfigured -MAX_DISK_SPACE=30GB +MAX_DISK_SPACE=300GB +BLOCKS=40 From 90c454ca11797dcd841281b8af05a43a15b91b43 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 May 2024 10:28:42 -0400 Subject: [PATCH 21/28] updated --- envs/dht1.cillium.dev.compute.agentartificial.com.txt | 2 +- envs/dht1.cillium.mixed.txt | 7 +++++++ envs/dht1.cillium.prod.compute.agentartificial.com.1.txt | 8 ++++++++ envs/dht1.cillium.prod.compute.agentartificial.com.txt | 5 +++++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 envs/dht1.cillium.mixed.txt create mode 100644 envs/dht1.cillium.prod.compute.agentartificial.com.1.txt diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index efbd26b98..c58b613c5 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -1,4 +1,4 @@ -INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 +INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/QmYUro5QJx3YvgC4A9UBXL3ESdb3wSHXZzqUL19Fmy5Gsp DEVICE=cuda #MODEL=Maykeye/TinyLLama-v0 MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B diff --git a/envs/dht1.cillium.mixed.txt b/envs/dht1.cillium.mixed.txt new file mode 100644 index 000000000..01288a9a1 --- /dev/null +++ b/envs/dht1.cillium.mixed.txt @@ -0,0 +1,7 @@ +INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/QmYUro5QJx3YvgC4A9UBXL3ESdb3wSHXZzqUL19Fmy5Gsp:/dns/dht1.cillium.prod.compute.agentartificial.com/tcp/8008/p2p/QmRuzeEHPohDR4BGvArrBrqUNC2YPvTc1m5iwTXefdKra8 +DEVICE=cuda +#MODEL=Maykeye/TinyLLama-v0 +MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B +PUBLIC_NAME=NameNotConfigured +MAX_DISK_SPACE=30GB +BLOCKS=1 diff --git a/envs/dht1.cillium.prod.compute.agentartificial.com.1.txt b/envs/dht1.cillium.prod.compute.agentartificial.com.1.txt new file mode 100644 index 000000000..7ae1551de --- /dev/null +++ b/envs/dht1.cillium.prod.compute.agentartificial.com.1.txt @@ -0,0 +1,8 @@ +INITIAL_PEERS=/dns/dht1.cillium.prod.compute.agentartificial.com/tcp/8008/p2p/QmRuzeEHPohDR4BGvArrBrqUNC2YPvTc1m5iwTXefdKra8 +DEVICE=cuda +#MODEL=Maykeye/TinyLLama-v0 +MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B +PUBLIC_NAME=NameNotConfigured +MAX_DISK_SPACE=30GB +BLOCKS=1 + diff --git a/envs/dht1.cillium.prod.compute.agentartificial.com.txt b/envs/dht1.cillium.prod.compute.agentartificial.com.txt index 6b922cd05..ef6826a80 100644 --- a/envs/dht1.cillium.prod.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.prod.compute.agentartificial.com.txt @@ -3,5 +3,10 @@ DEVICE=cuda #MODEL=Maykeye/TinyLLama-v0 MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B PUBLIC_NAME=NameNotConfigured +<<<<<<< HEAD MAX_DISK_SPACE=300GB BLOCKS=40 +======= +MAX_DISK_SPACE=30GB +BLOCKS=1 +>>>>>>> e56d9ce (blocks) From 04132bdca7a610877e102da99eb4be84cbaf0b4a Mon Sep 17 00:00:00 2001 From: j mike dupont Date: Tue, 30 Apr 2024 16:28:03 +0000 Subject: [PATCH 22/28] update port --- docker-compose.yml | 8 ++++---- envs/dht1.cillium.dev.compute.agentartificial.com.txt | 11 ++++++++--- .../dht1.cillium.dev.compute.agentartificial.com.txt | 7 +++++++ setup.sh | 3 ++- 4 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt diff --git a/docker-compose.yml b/docker-compose.yml index 89b11cfbb..58d35d334 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,11 +10,11 @@ services: - INITIAL_PEERS=${INITIAL_PEERS} - DEVICE=${DEVICE} - BLOCKS=${BLOCKS} - command: python -m petals.cli.run_server --port 31331 --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE - # --max_disk_space $MAX_DISK_SPACE - # --public-name $PUBLIC_NAME + - PORT=${PORT} + command: python -m petals.cli.run_server --port ${PORT} --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE + ports: - - "31331:31331" + - "$PORT:$PORT" restart: always deploy: resources: diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index c58b613c5..c0ec4ceee 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -1,7 +1,12 @@ INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/QmYUro5QJx3YvgC4A9UBXL3ESdb3wSHXZzqUL19Fmy5Gsp DEVICE=cuda -#MODEL=Maykeye/TinyLLama-v0 -MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B + PUBLIC_NAME=NameNotConfigured MAX_DISK_SPACE=30GB -BLOCKS=30 +BLOCKS=2 +#BLOCKS=33 + +MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct +#MODEL=Maykeye/TinyLLama-v0 +#MODEL=SanjiWatsuki/TinyMixtral-32x248M + diff --git a/envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt new file mode 100644 index 000000000..be8f62210 --- /dev/null +++ b/envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt @@ -0,0 +1,7 @@ +INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/Qmb3skfrki1PR8ww6nxvoGm51F5imK3e1DPMZgtay6ofE2 +DEVICE=cuda +MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct +BLOCKS=100 +#MODEL=Maykeye/TinyLLama-v0 +#MODEL=SanjiWatsuki/TinyMixtral-32x248M +PORT=31332 diff --git a/setup.sh b/setup.sh index 990601647..830da0be2 100644 --- a/setup.sh +++ b/setup.sh @@ -2,4 +2,5 @@ set -e -docker compose --profile miner --env-file envs/dht1.cillium.dev.compute.agentartificial.com.txt up -d +docker compose --profile miner --env-file envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt up -d +#sudo docker compose --profile miner --env-file envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt up -d From dea21b5fd83c67d01562cb36f64d603486c25ed6 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 2 May 2024 16:11:11 -0400 Subject: [PATCH 23/28] changing defaults to set disk space --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 58d35d334..f2c7c5672 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - DEVICE=${DEVICE} - BLOCKS=${BLOCKS} - PORT=${PORT} - command: python -m petals.cli.run_server --port ${PORT} --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public-name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE ports: - "$PORT:$PORT" From cf70fc1192f460f69f871ce005c31c6a5d7221bd Mon Sep 17 00:00:00 2001 From: mike dupont Date: Wed, 8 May 2024 15:34:42 -0400 Subject: [PATCH 24/28] sauerkraut --- envs/dht1.cillium.dev.compute.agentartificial.com.txt | 5 ----- envs/dht1.cillium.prod.compute.agentartificial.com.txt | 9 ++------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index c0ec4ceee..5fd57a5df 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -1,12 +1,7 @@ INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/QmYUro5QJx3YvgC4A9UBXL3ESdb3wSHXZzqUL19Fmy5Gsp DEVICE=cuda - PUBLIC_NAME=NameNotConfigured MAX_DISK_SPACE=30GB BLOCKS=2 #BLOCKS=33 - MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct -#MODEL=Maykeye/TinyLLama-v0 -#MODEL=SanjiWatsuki/TinyMixtral-32x248M - diff --git a/envs/dht1.cillium.prod.compute.agentartificial.com.txt b/envs/dht1.cillium.prod.compute.agentartificial.com.txt index ef6826a80..eb1972556 100644 --- a/envs/dht1.cillium.prod.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.prod.compute.agentartificial.com.txt @@ -1,12 +1,7 @@ INITIAL_PEERS=/dns/dht1.cillium.prod.compute.agentartificial.com/tcp/8008/p2p/QmRuzeEHPohDR4BGvArrBrqUNC2YPvTc1m5iwTXefdKra8 DEVICE=cuda -#MODEL=Maykeye/TinyLLama-v0 -MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B PUBLIC_NAME=NameNotConfigured -<<<<<<< HEAD MAX_DISK_SPACE=300GB -BLOCKS=40 -======= +#BLOCKS=40 +BLOCKS=2 MAX_DISK_SPACE=30GB -BLOCKS=1 ->>>>>>> e56d9ce (blocks) From 7f90c068d46bc2b1fd3a8b916af82e1dc3877cc1 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Thu, 9 May 2024 14:12:09 -0400 Subject: [PATCH 25/28] changing setings --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 830da0be2..b9b40da3c 100644 --- a/setup.sh +++ b/setup.sh @@ -2,5 +2,5 @@ set -e -docker compose --profile miner --env-file envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt up -d +docker compose --profile miner --env-file envs/dht1.cillium.dev.compute.agentartificial.com.txt up -d #sudo docker compose --profile miner --env-file envs/gpu/h100/dht1.cillium.dev.compute.agentartificial.com.txt up -d From e4581cf58910f7aca69f6fa090d93c9f062080de Mon Sep 17 00:00:00 2001 From: Mike DuPont Date: Mon, 13 May 2024 03:51:18 -0400 Subject: [PATCH 26/28] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f2c7c5672..b389f5f22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - DEVICE=${DEVICE} - BLOCKS=${BLOCKS} - PORT=${PORT} - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public-name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE + command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public_name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE ports: - "$PORT:$PORT" From 9caa5b6e2cf2748cd66f7578fb97e3879f485704 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Sat, 25 May 2024 05:49:04 -0400 Subject: [PATCH 27/28] bugfix --- docker-compose.yml | 5 +---- envs/dht1.cillium.dev.compute.agentartificial.com.txt | 2 +- setup.sh | 0 3 files changed, 2 insertions(+), 5 deletions(-) mode change 100644 => 100755 setup.sh diff --git a/docker-compose.yml b/docker-compose.yml index b389f5f22..a3dd3a612 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,11 +10,8 @@ services: - INITIAL_PEERS=${INITIAL_PEERS} - DEVICE=${DEVICE} - BLOCKS=${BLOCKS} - - PORT=${PORT} - command: python -m petals.cli.run_server --port 31331 --num_blocks=1 $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public_name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE + command: python -m petals.cli.run_server --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public_name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE - ports: - - "$PORT:$PORT" restart: always deploy: resources: diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index 5fd57a5df..cd1ba3e80 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -2,6 +2,6 @@ INITIAL_PEERS=/dns/dht1.cillium.dev.compute.agentartificial.com/tcp/8008/p2p/QmY DEVICE=cuda PUBLIC_NAME=NameNotConfigured MAX_DISK_SPACE=30GB -BLOCKS=2 +BLOCKS=3 #BLOCKS=33 MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct diff --git a/setup.sh b/setup.sh old mode 100644 new mode 100755 From 17287db9e526b92d1bb215a2ba552ee05e987cb3 Mon Sep 17 00:00:00 2001 From: mike dupont Date: Mon, 3 Jun 2024 19:20:35 -0400 Subject: [PATCH 28/28] public ip --- docker-compose.yml | 12 +++++++++++- .../dht1.cillium.dev.compute.agentartificial.com.txt | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a3dd3a612..dca6cb09b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,16 @@ services: - INITIAL_PEERS=${INITIAL_PEERS} - DEVICE=${DEVICE} - BLOCKS=${BLOCKS} - command: python -m petals.cli.run_server --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public_name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE + - PORT=${PORT} + - PUBLIC_IP=${PUBLIC_IP} +# - GOLOG_LOG_LEVEL=debug + - GOLOG_LOG_LEVEL=info + - GOLOG_FILE=/opt/cillium/var/log/p2pd_miner.log + + command: python -m petals.cli.run_server --num_blocks=$BLOCKS $MODEL --initial_peers $INITIAL_PEERS --device=$DEVICE --public_name $PUBLIC_NAME --max_disk_space $MAX_DISK_SPACE --public_ip=${PUBLIC_IP} --port ${PORT} --identity_path /cache/bootstrap1.id + volumes: + - petals-cache-backbone:/cache + - log:/opt/cillium/var/log restart: always deploy: @@ -20,5 +29,6 @@ services: - driver: nvidia count: all capabilities: [gpu] + volumes: petals-cache-backbone: diff --git a/envs/dht1.cillium.dev.compute.agentartificial.com.txt b/envs/dht1.cillium.dev.compute.agentartificial.com.txt index cd1ba3e80..32a0d303c 100644 --- a/envs/dht1.cillium.dev.compute.agentartificial.com.txt +++ b/envs/dht1.cillium.dev.compute.agentartificial.com.txt @@ -5,3 +5,5 @@ MAX_DISK_SPACE=30GB BLOCKS=3 #BLOCKS=33 MODEL=VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct +PORT=30331 +PUBLIC_IP=FIXME