setup-llama-linux.sh

#!/usr/bin/env bash

set -euo pipefail

# 国内环境默认使用 HuggingFace 镜像站，可通过环境变量覆盖

export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"

# GitHub 国内加速代理，可通过环境变量覆盖或置空

GITHUB_PROXY="${GITHUB_PROXY:-https://ghproxy.com/}"

# 用法:

# chmod +x setup-llama-linux.sh

# ./setup-llama-linux.sh [huggingface模型ID[:GGUF后缀]]

# 示例:

# ./setup-llama-linux.sh

# ./setup-llama-linux.sh Andycurrent/Qwen3.5-4B-Uncensored-HauhauCS-Aggressive:Q4_K_M

# ./setup-llama-linux.sh unsloth/Llama-3.2-1B-Instruct-GGUF:Q4_K_M

MODEL_SPEC="${1:-Andycurrent/Qwen3.5-4B-Uncensored-HauhauCS-Aggressive:Q4_K_M}"

MODEL_ID="${MODEL_SPEC%%:*}"

GGUF_SUFFIX="${MODEL_SPEC##*:}"

# 如果只给了模型 ID 没给后缀，默认 Q4_K_M

if [[ "$MODEL_ID" == "$MODEL_SPEC" ]]; then

GGUF_SUFFIX="${2:-Q4_K_M}"

if [[ -z "$MODEL_ID" ]]; then

echo "错误: 请提供 Hugging Face 模型 ID"

echo "用法: $0 <huggingface模型ID>[:GGUF后缀]"

echo "示例: $0 Andycurrent/Qwen3.5-4B-Uncensored-HauhauCS-Aggressive:Q4_K_M"

exit 1

MODEL_DIR="models"

# 1. 选择后端（三种模式都使用 llama.cpp 预编译 release）

echo ""

echo "请选择运行后端:"

echo " 1) CPU 模式 (默认，无需 GPU 驱动)"

echo " 2) Vulkan 模式 (推荐，兼容 AMD/Intel/NVIDIA 核显/独显)"

echo " 3) ROCm/HIP 模式 (AMD GPU 专用，性能通常最好)"

read -rp "输入 1/2/3 [默认 1]: " BACKEND

BACKEND="${BACKEND:-1}"

case "$BACKEND" in

1) BACKEND_NAME="cpu"; BACKEND_DIR="llama.cpp-cpu-bin" ;;

2) BACKEND_NAME="vulkan"; BACKEND_DIR="llama.cpp-vulkan-bin" ;;

3) BACKEND_NAME="rocm"; BACKEND_DIR="llama.cpp-rocm-bin" ;;

*) echo "错误: 无效选择"; exit 1 ;;

esac

# 2. 安装依赖（三种模式都只需要下载/解压工具）

echo "==> 安装下载/解压所需依赖..."

if command -v apt-get &>/dev/null; then

sudo apt-get update

sudo apt-get install -y python3-pip wget curl tar

else

echo "警告: 未检测到 apt-get，请手动安装 python3-pip、wget、curl、tar"

# 3. 后端特定检查

if [[ "$BACKEND" == "2" ]]; then

echo "==> Vulkan 模式：安装 Vulkan 驱动..."

if command -v apt-get &>/dev/null; then

sudo apt-get install -y vulkan-tools mesa-vulkan-drivers

else

echo "警告: 未检测到 apt-get，请手动安装 vulkan-tools、mesa-vulkan-drivers"

if command -v vulkaninfo &>/dev/null; then

echo "==> Vulkan 设备信息："

vulkaninfo --summary 2>/dev/null || true

else

echo "提示: 未找到 vulkaninfo，安装完成后建议重启或重新登录以加载驱动。"

elif [[ "$BACKEND" == "3" ]]; then

echo "==> ROCm/HIP 模式：检查 ROCm runtime..."

if ! command -v rocminfo &>/dev/null && ! command -v hipcc &>/dev/null; then

echo "错误: 未检测到 rocminfo / hipcc，请先安装 ROCm runtime。"

echo "参考命令:"

echo " sudo apt install rocm-dev hipblas rocblas amdgpu-install"

exit 1

export HSA_OVERRIDE_GFX_VERSION="${HSA_OVERRIDE_GFX_VERSION:-11.0.3}"

export AMDGPU_TARGETS="${AMDGPU_TARGETS:-gfx1103}"

echo "==> HSA_OVERRIDE_GFX_VERSION=$HSA_OVERRIDE_GFX_VERSION"

echo "==> AMDGPU_TARGETS=$AMDGPU_TARGETS"

# 4. 下载并解压 llama.cpp 预编译包

echo "==> 查询 llama.cpp 最新 ${BACKEND_NAME} 预编译包..."

RELEASE_JSON=$(mktemp)

trap 'rm -f "$RELEASE_JSON"' EXIT

if ! curl -fsSL -o "$RELEASE_JSON" "${GITHUB_PROXY}https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"; then

echo "错误: 无法获取 llama.cpp release 信息，请检查网络或 GITHUB_PROXY 设置"

exit 1

LATEST_TAG=$(python3 -c "import json; print(json.load(open('$RELEASE_JSON'))['tag_name'])")

ASSET=$(python3 -c "

import json

name = ''

for asset in json.load(open('$RELEASE_JSON'))['assets']:

n = asset['name']

if not n.endswith('.tar.gz'):

continue

if '$BACKEND_NAME' == 'cpu' and n.endswith('-bin-ubuntu-x64.tar.gz'):

name = n

break

elif '$BACKEND_NAME' == 'vulkan' and '-bin-ubuntu-vulkan-' in n:

name = n

break

elif '$BACKEND_NAME' == 'rocm' and '-bin-ubuntu-rocm-' in n:

name = n

break

print(name)

if [[ -z "$LATEST_TAG" || -z "$ASSET" ]]; then

echo "错误: 未找到 ${BACKEND_NAME} 预编译包"

exit 1

echo "==> 最新版本: $LATEST_TAG"

echo "==> 下载包: $ASSET"

rm -rf "$BACKEND_DIR"

mkdir -p "$BACKEND_DIR"

DOWNLOAD_URL="${GITHUB_PROXY}https://github.com/ggml-org/llama.cpp/releases/download/${LATEST_TAG}/${ASSET}"

echo "==> 下载地址: $DOWNLOAD_URL"

wget -q --show-progress -O "$ASSET" "$DOWNLOAD_URL"

echo "==> 解压到 $BACKEND_DIR ..."

tar -xzf "$ASSET" -C "$BACKEND_DIR" --strip-components=1

rm -f "$ASSET"

# 确定可执行文件路径

find_llama_bin() {

local name="$1"

if [[ -f "$BACKEND_DIR/bin/$name" ]]; then

echo "$BACKEND_DIR/bin/$name"

elif [[ -f "$BACKEND_DIR/build/bin/$name" ]]; then

echo "$BACKEND_DIR/build/bin/$name"

else

echo ""

}

CLI_BIN=$(find_llama_bin llama-cli)

SERVER_BIN=$(find_llama_bin llama-server)

if [[ -z "$CLI_BIN" || -z "$SERVER_BIN" ]]; then

echo "错误: 解压后未找到 llama-cli / llama-server"

exit 1

# 5. 安装 huggingface-cli 用于下载模型

echo "==> 安装 huggingface-hub..."

pip3 install --user -U huggingface-hub

# 6. 下载模型

mkdir -p "$MODEL_DIR"

echo "==> 使用 HuggingFace 镜像: $HF_ENDPOINT"

echo "==> 下载模型 $MODEL_ID (后缀: $GGUF_SUFFIX)..."

huggingface-cli download "$MODEL_ID" \

--include "*.gguf" \

--local-dir "$MODEL_DIR/$MODEL_ID" \

--local-dir-use-symlinks False

# 尝试匹配对应后缀的 GGUF 文件

MODEL_FILE=$(find "$MODEL_DIR/$MODEL_ID" -maxdepth 1 -type f -name "*${GGUF_SUFFIX}*.gguf" | head -n 1)

# 如果没找到指定后缀，则任选第一个 GGUF

if [[ -z "$MODEL_FILE" ]]; then

MODEL_FILE=$(find "$MODEL_DIR/$MODEL_ID" -maxdepth 1 -type f -name "*.gguf" | head -n 1)

if [[ -z "$MODEL_FILE" ]]; then

echo "错误: 未找到任何 .gguf 文件"

exit 1

echo "==> 使用模型文件: $MODEL_FILE"

# 7. 选择运行模式并执行

echo ""

echo "请选择运行模式:"

echo " 1) cli 模式 (命令行交互)"

echo " 2) server 模式 (HTTP API 服务，默认端口 8080)"

read -rp "输入 1 或 2 [默认 1]: " MODE

MODE="${MODE:-1}"

# 公共参数

COMMON_ARGS=(

--model "$MODEL_FILE"

--ctx-size 512

--threads "$(nproc)"

--no-mmap

--cache-type-k q4_0

--cache-type-v q4_0

)

# 根据后端决定是否 GPU offload

if [[ "$BACKEND" == "1" ]]; then

COMMON_ARGS+=(-ngl 0)

else

COMMON_ARGS+=(-ngl 999)

if [[ "$MODE" == "2" ]]; then

BIN="$SERVER_BIN"

RUN_ARGS=("${COMMON_ARGS[@]}" --host 127.0.0.1 --port 8080)

echo ""

echo "==> 准备以 server 模式启动..."

else

BIN="$CLI_BIN"

RUN_ARGS=("${COMMON_ARGS[@]}" --prompt "Hello, world!" --n-predict 64)

echo ""

echo "==> 准备以 cli 模式启动..."

# 打印将要执行的完整命令

echo ""

echo "将要执行的命令:"

printf '%q ' "$BIN" "${RUN_ARGS[@]}"

echo ""

# 执行

"$BIN" "${RUN_ARGS[@]}"