bash <(wget -qO- https://xuanyuan.cloud/docker.sh)

docker --version  # 需≥20.10
docker-compose --version  # 需≥v2.0

# 配置 NVIDIA 源
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
curl -s -L https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list

# 安装工具包
sudo apt update && sudo apt install -y nvidia-container-toolkit

# 配置 Docker 并重启
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# 配置 NVIDIA 源
sudo dnf config-manager --add-repo https://nvidia.github.io/libnvidia-container/rhel8/libnvidia-container.repo

# 安装工具包
sudo dnf install -y nvidia-container-toolkit

# 配置 Docker 并重启
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

docker info | grep -i "nvidia"  # 预期输出包含 "nvidia" 运行时
docker run --rm --runtime=nvidia --gpus all nvidia/cuda:12.2.0-base nvidia-smi  # 预期输出 GPU 信息

docker pull xxx.xuanyuan.run/ai/qwen3:latest

docker pull xxx.xuanyuan.run/ai/qwen3:0.6B-Q4_K_M

┌─────────────┐
│ Developer   │
│ curl / App  │
└──────┬──────┘
       │ HTTP 127.0.0.1:8080
       ▼
┌────────────────────────┐
│ QWEN3 Docker Container │
│ - 单实例               │
│ - 可 root              │
│ - GPU / CPU            │
└─────────┬──────────────┘
          │
          ▼
   Docker Volume
   - 模型权重缓存
   - 推理缓存

┌─────────────┐
│ Client/App  │
└──────┬──────┘
       │ HTTPS :443
       ▼
┌──────────────────┐
│ Nginx / LB       │
│ - TLS            │
│ - API Key 鉴权    │
│ - 限流           │
└──────┬───────────┘
       │
 ┌─────┴───────────────┐
 │                     │
 ▼                     ▼
┌──────────────┐  ┌──────────────┐
│ QWEN3 实例 A │  │ QWEN3 实例 B │
│ 非 root      │  │ 非 root      │
│ GPU 推理     │  │ GPU 推理     │
│ HealthCheck  │  │ HealthCheck  │
└──────┬───────┘  └──────┬───────┘
       │                 │
       └───────共享存储 / 模型缓存

docker run -d \
  --name qwen3-test \
  --gpus all \  # 自动识别并使用所有 GPU（无 GPU 可移除）-p 127.0.0.1:8080:8080 \  # 仅绑定本地回环地址，避免公网暴露
  -e ENABLE_THINKING=true \
  -v qwen3-data:/app/data \
  --restart unless-stopped \
  xxx.xuanyuan.run/ai/qwen3:latest

docker run -d \
  --name qwen3-prod \
  --gpus all \
  --runtime=nvidia \  # 兼容非 Swarm 模式的 GPU 映射
  -e NVIDIA_VISIBLE_DEVICES=all \
  -e NVIDIA_DRIVER_CAPABILITIES=compute,utility \
  -p 127.0.0.1:8080:8080 \
  -e ENABLE_THINKING=true \
  -e MAX_CONCURRENT_REQUESTS=10 \
  -e CONTEXT_WINDOW_SIZE=41000 \
  # 配置挂载：无需自定义配置可省略此行
  -v /path/to/local/config:/app/config \
  -v qwen3-data:/app/data \
  # 只读文件系统 + 临时目录（解决 GPU/ 模型缓存写权限）--read-only \
  --tmpfs /tmp:rw,size=1g \
  # 安全加固
  --user 1000:1000 \  # 非 root 用户运行（需确保宿主机 1000 用户存在）--cap-drop ALL \  # 移除所有 Linux 能力
  --security-opt no-new-privileges:true \
  # 日志配置（企业级推荐）--log-driver=json-file \
  --log-opt max-size=100m \
  --log-opt max-file=7 \
  # 资源限制
  --memory=16g \
  --memory-swap=16g \
  --cpus=4 \
  # 健康检查（基于真实可用接口）--health-cmd "curl -f http://localhost:8080/v1/models || exit 1" \
  --health-interval 30s \
  --health-timeout 10s \
  --health-retries 3 \
  --health-start-period 60s \
  # 高可用
  --restart unless-stopped \
  xxx.xuanyuan.run/ai/qwen3:latest

version: '3.8'

services:
  qwen3:
    image: xxx.xuanyuan.run/ai/qwen3:8B-Q4_K_M
    container_name: qwen3-prod
    restart: unless-stopped
    # GPU 配置（兼容非 Swarm 模式）runtime: nvidia
    deploy:
      # 仅 Swarm 模式生效，非 Swarm 可忽略
      resources:
        limits:
          cpus: '4'
          memory: 16G
          reservations:
            devices:
              - driver: nvidia
                count: all
                capabilities: [gpu]
    # 网络配置（仅绑定本地地址）ports:
      - "127.0.0.1:8080:8080"
    # 环境变量
    environment:
      - ENABLE_THINKING=true
      - MAX_CONCURRENT_REQUESTS=10
      - CONTEXT_WINDOW_SIZE=41000
      - NVIDIA_VISIBLE_DEVICES=all
      - NVIDIA_DRIVER_CAPABILITIES=compute,utility
    # 数据持久化
    volumes:
      # 无需自定义配置可删除此行
      - /path/to/local/config:/app/config
      - qwen3-data:/app/data
    # 临时目录（只读模式必需）tmpfs:
      - /tmp:rw,size=1g
    # 安全加固
    user: "1000:1000"
    read_only: true
    cap_drop:
      - ALL
    security_opt:
      - no-new-privileges:true
    # 日志配置
    logging:
      driver: "json-file"
      options:
        max-size: "100m"
        max-file: "7"
    # 健康检查
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/v1/models"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

volumes:
  qwen3-data:
    driver: local

docker-compose up -d

# 检查运行状态
docker ps | grep qwen3-prod

# 检查健康状态
docker inspect -f '{{.State.Health.Status}}' qwen3-prod  # 预期输出：healthy

# 查看实时日志
docker logs -f qwen3-prod | grep "Service started successfully"

curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"qwen3-8b","messages": [{"role":"user","content":"What is the square root of 144?"}]
  }'

{
  "id": "chat-xxx",
  "object": "chat.completion",
  "created": 1728567890,
  "model": "qwen3-8b",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "The square root of 144 is 12."
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 15,
    "completion_tokens": 10,
    "total_tokens": 25
  }
}

curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"qwen3-8b","messages": [{"role":"user","content":"/think Solve: A train travels 120 km in 2 hours. What is its average speed in m/s?"}]
  }'

{
  "id": "chat-yyy",
  "object": "chat.completion",
  "created": 1728567900,
  "model": "qwen3-8b",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "To find the average speed in m/s, follow these steps:\n1. Calculate speed in km/h: Distance = 120 km, Time = 2 hours → Speed = 120/2 = 60 km/h.\n2. Convert km/h to m/s: Multiply by 1000/3600 (or 5/18) → 60 × (5/18) ≈ 16.67 m/s.\n\nAverage speed: 16.67 m/s."
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 30,
    "completion_tokens": 80,
    "total_tokens": 110
  }
}

curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"qwen3-8b","messages": [{"role":"user","content":" 日本の首都はどこですか？"}]
  }'

{
  "choices": [
    {
      "message": {
        "content": "日本の首都は東京です。",
        "role": "assistant"
      }
    }
  ]
}

curl -X POST http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model":"qwen3-8b","messages": [{"role":"user","content":"What is the current weather in Beijing? Use the weather API."}]
  }'

{
  "choices": [
    {
      "message": {
        "content": "","role":"assistant","tool_calls": [
          {
            "function": {
              "name": "get_weather",
              "parameters": {"city": "Beijing"}
            },
            "type": "function"
          }
        ]
      }
    }
  ]
}

# 安装 hey
go install github.com/rakyll/hey@latest

# 压测命令（8B 模型建议 QPS≤5）hey -n 100 -c 5 -m POST -H "Content-Type: application/json" -d '{"model":"qwen3-8b","messages":[{"role":"user","content":"Hello, world!"}]}' http://localhost:8080/v1/chat/completions

# qwen3-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: qwen3-8b
  namespace: llm
spec:
  replicas: 2
  selector:
    matchLabels:
      app: qwen3-8b
  template:
    metadata:
      labels:
        app: qwen3-8b
    spec:
      # GPU 节点选择
      nodeSelector:
        nvidia.com/gpu.present: "true"
      # 安全上下文
      securityContext:
        runAsUser: 1000
        runAsGroup: 1000
        readOnlyRootFilesystem: true
        allowPrivilegeEscalation: false
        capabilities:
          drop: ["ALL"]
      # 容器配置
      containers:
      - name: qwen3-8b
        image: xxx.xuanyuan.run/ai/qwen3:8B-Q4_K_M
        ports:
        - containerPort: 8080
        env:
        - name: ENABLE_THINKING
          value: "true"
        - name: MAX_CONCURRENT_REQUESTS
          value: "10"
        # GPU 资源请求
        resources:
          requests:
            cpu: 4
            memory: 16Gi
            nvidia.com/gpu: 1
          limits:
            cpu: 4
            memory: 16Gi
            nvidia.com/gpu: 1
        # 健康检查
        livenessProbe:
          httpGet:
            path: /v1/models
            port: 8080
          initialDelaySeconds: 60
          periodSeconds: 30
        readinessProbe:
          httpGet:
            path: /v1/models
            port: 8080
          initialDelaySeconds: 30
          periodSeconds: 10
        # 数据卷
        volumeMounts:
        - name: qwen3-data
          mountPath: /app/data
        # 临时目录
        - name: tmp
          mountPath: /tmp
      volumes:
      - name: qwen3-data
        persistentVolumeClaim:
          claimName: qwen3-data-pvc
      - name: tmp
        emptyDir:
          medium: Memory
          sizeLimit: 1Gi