五种搭建LLM服务的方法和代码示例

pip install llama-cpp-python[server] \
  --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu

mkdir -p models/7B
 wget -O models/7B/llama-2-7b-chat.Q5_K_M.gguf https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf?download=true

python3 -m llama_cpp.server --model models/7B/llama-2-7b-chat.Q5_K_M.gguf

messages=[
    {"role": "system", "content": "You are a helpful assistant."},
    {
        "role": "user",
        "content": "What are the names of the four main characters of South Park?",
    },
 ]

pip install vllm

python -m vllm.entrypoints.openai.api_server --model TheBloke/Llama-2-7B-Chat-AWQ --api-key DEFAULT --quantization awq --enforce-eager

sudo apt install nvidia-cuda-toolkit

distributinotallow=$(. /etc/os-release;echo $ID$VERSION_ID)
 curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
 curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
 
 sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
 sudo systemctl restart docker

sudo tee /etc/docker/daemon.json <<EOF
 {
    "runtimes": {
        "nvidia": {
            "path": "/usr/bin/nvidia-container-runtime",
            "runtimeArgs": []
        }
    }
 }
 EOF
 sudo pkill -SIGHUP dockerd

docker run --runtime nvidia --gpus all \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    -p 8000:8000 \
    --ipc=host \
    vllm/vllm-openai:latest \
    --model TheBloke/Llama-2-7B-Chat-AWQ \
    --quantization awq --enforce-eager

pip install modal

modal setup

# https://modal.com/docs/examples/vllm_mixtral
 import os 
 import time
 
 from modal import Image, Stub, enter, exit, gpu, method
 
 APP_NAME = "example-vllm-llama-chat"
 MODEL_DIR = "/model"
 BASE_MODEL = "TheBloke/Llama-2-7B-Chat-AWQ"
 GPU_CONFIG = gpu.T4(count=1)

vllm_image = ( # https://modal.com/docs/examples/vllm_mixtral
    Image.from_registry("nvidia/cuda:12.1.1-devel-ubuntu22.04", add_pythnotallow="3.10")
    .pip_install(
        "vllm==0.3.2",
        "huggingface_hub==0.19.4",
        "hf-transfer==0.1.4",
        "torch==2.1.2",
    )
    .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
    .run_function(download_model_to_folder, timeout=60 * 20)
 )

stub = Stub(APP_NAME)

class Model:  
    @enter() # Lifecycle functions
    def start_engine(self):
        import time
 
        from vllm.engine.arg_utils import AsyncEngineArgs
        from vllm.engine.async_llm_engine import AsyncLLMEngine
 
        print("? cold starting inference")
        start = time.monotonic_ns()
 
        engine_args = AsyncEngineArgs(
            model=MODEL_DIR,
            tensor_parallel_size=GPU_CONFIG.count,
            gpu_memory_utilizatinotallow=0.90,
            enforce_eager=False, # capture the graph for faster inference, but slower cold starts
            disable_log_stats=True, # disable logging so we can stream tokens

@stub.function()
 def generate(user_question: str):
    model = Model()
 
    print("Sending new request:", user_question, "\n\n")
 
    result = ""
    for text in model.completion_stream.remote_gen(user_question):
        print(text, end="", flush=True)
        result += text
 
    return result

modal deploy vllm_modal_deploy.py

import timeit
 
 import modal
 
 APP_NAME = "example-vllm-llama-chat"
 f = modal.Function.lookup(APP_NAME, "generate")
 
 start_time = timeit.default_timer()
 
 print(f.remote("What are the names of the four main characters of South park?"))
 
 elapsed = timeit.default_timer() - start_time
 
 print(f"{elapsed=}")

export API_KEY="CHANGEME"
 export MODEL="meta-llama/Llama-2-7b-chat-hf"
 export BASE_URL="https://api.endpoints.anyscale.com/v1"
 python openai_client.py