#!/usr/bin/env bash set -u LLAMA_BIN="${LLAMA_BIN:-/home/hotaisle/llama.cpp/build/bin/llama-completion}" MODEL_GLOB="${MODEL_GLOB:-*.gguf}" PROMPT="${PROMPT:-Reply with this exact phrase: crazy fish swims quick}" SYSTEM_PROMPT="${SYSTEM_PROMPT:-You are a precise assistant. Repeat the requested phrase exactly and nothing else.}" N_PREDICT="${N_PREDICT:-32}" CTX_SIZE="${CTX_SIZE:-2048}" BATCH_SIZE="${BATCH_SIZE:-512}" THREADS="${THREADS:-8}" OUTDIR="${OUTDIR:-test-logs}" TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-0}" WORKERS="${WORKERS:-4}" GPU_IDS="${GPU_IDS:-0 1 2 3}" GPU_LAYERS="${GPU_LAYERS:-all}" USE_GPU="${USE_GPU:-auto}" GGUF_PY_PATH="${GGUF_PY_PATH:-/home/hotaisle/llama.cpp/gguf-py}" if [[ ! -x "$LLAMA_BIN" ]]; then echo "error: llama binary not found or not executable: $LLAMA_BIN" >&2 exit 1 fi if ! command -v python3 >/dev/null 2>&1; then echo "error: python3 is required to render chat templates" >&2 exit 1 fi mkdir -p "$OUTDIR" shopt -s nullglob models=( $MODEL_GLOB ) shopt -u nullglob if [[ ${#models[@]} -eq 0 ]]; then echo "error: no models matched $MODEL_GLOB in $(pwd)" >&2 exit 1 fi # shellcheck disable=SC2206 gpu_ids=( $GPU_IDS ) if (( WORKERS <= 0 )); then echo "error: WORKERS must be > 0" >&2 exit 1 fi if (( ${#gpu_ids[@]} == 0 )); then echo "error: GPU_IDS did not parse into any GPU indices" >&2 exit 1 fi if (( WORKERS > ${#gpu_ids[@]} )); then echo "error: WORKERS=$WORKERS but only ${#gpu_ids[@]} GPU IDs were provided: $GPU_IDS" >&2 exit 1 fi have_visible_devices=0 if "$LLAMA_BIN" --list-devices 2>/dev/null | awk 'BEGIN{found=0} /^Available devices:/{next} NF{found=1} END{exit found?0:1}'; then have_visible_devices=1 fi case "$USE_GPU" in auto) if (( have_visible_devices == 1 )); then use_gpu=1 else use_gpu=0 fi ;; 1|true|TRUE|yes|YES|on|ON) use_gpu=1 ;; 0|false|FALSE|no|NO|off|OFF) use_gpu=0 ;; *) echo "error: USE_GPU must be auto, true, or false" >&2 exit 1 ;; esac timestamp="$(date -u +%Y%m%dT%H%M%SZ)" summary_file="$OUTDIR/summary-$timestamp.txt" status_dir="$OUTDIR/status-$timestamp" mkdir -p "$status_dir" echo "llama_bin: $LLAMA_BIN" | tee "$summary_file" echo "models: ${#models[@]}" | tee -a "$summary_file" echo "workers: $WORKERS" | tee -a "$summary_file" echo "gpu_ids: ${gpu_ids[*]}" | tee -a "$summary_file" echo "use_gpu: $use_gpu (USE_GPU=$USE_GPU, visible_devices=$have_visible_devices)" | tee -a "$summary_file" echo "system_prompt: $SYSTEM_PROMPT" | tee -a "$summary_file" echo "prompt: $PROMPT" | tee -a "$summary_file" echo "n_predict: $N_PREDICT, ctx_size: $CTX_SIZE, batch_size: $BATCH_SIZE, threads: $THREADS" | tee -a "$summary_file" if (( use_gpu == 1 )); then echo "gpu_layers: $GPU_LAYERS" | tee -a "$summary_file" fi echo | tee -a "$summary_file" render_chat_prompt() { local model="$1" local rendered_prompt_file="$2" python3 - "$model" "$rendered_prompt_file" "$SYSTEM_PROMPT" "$PROMPT" "$GGUF_PY_PATH" <<'PY' import json import sys from jinja2.sandbox import ImmutableSandboxedEnvironment model_path, output_path, system_prompt, user_prompt, gguf_py_path = sys.argv[1:] sys.path.insert(0, gguf_py_path) from gguf import GGUFReader reader = GGUFReader(model_path) field = reader.fields["tokenizer.chat_template"] template = bytes(field.parts[field.data[0]]).decode("utf-8") env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True) env.globals["raise_exception"] = lambda msg: (_ for _ in ()).throw(RuntimeError(msg)) env.filters["tojson"] = lambda obj, indent=None: json.dumps(obj, ensure_ascii=False, indent=indent) messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ] rendered = env.from_string(template).render( messages=messages, tools=[], add_generation_prompt=True, enable_thinking=False, add_vision_id=False, bos_token="", eos_token="", ) with open(output_path, "w", encoding="utf-8") as f: f.write(rendered) PY } run_one() { local model="$1" local gpu_id="$2" local slot="$3" local model_name log_file status_file output_file rendered_prompt_file status lowered_output generated_output local -a cmd run_cmd extra_args model_name="$(basename "$model" .gguf)" log_file="$OUTDIR/${model_name}-${timestamp}.log" status_file="$status_dir/${model_name}.status" output_file="$OUTDIR/${model_name}-${timestamp}.out" rendered_prompt_file="$OUTDIR/${model_name}-${timestamp}.prompt" if ! render_chat_prompt "$model" "$rendered_prompt_file" >>"$log_file" 2>&1; then status=90 { echo "model=$model" echo "gpu_id=$gpu_id" echo "worker_slot=$slot" echo "use_gpu=$use_gpu" echo "log_file=$log_file" echo "output_file=$output_file" echo "exit_code=$status" } >"$status_file" return fi cmd=( "$LLAMA_BIN" --model "$model" --file "$rendered_prompt_file" --n-predict "$N_PREDICT" --ctx-size "$CTX_SIZE" --batch-size "$BATCH_SIZE" --threads "$THREADS" --temp 0 --top-k 1 --top-p 1.0 -no-cnv --no-display-prompt --simple-io --no-warmup ) if (( use_gpu == 1 )); then cmd+=( --split-mode none --gpu-layers "$GPU_LAYERS" --fit on ) else cmd+=( --fit off ) fi if [[ -n "${LLAMA_EXTRA_ARGS:-}" ]]; then # Intentionally split on shell words so callers can pass normal CLI flags. # shellcheck disable=SC2206 extra_args=( ${LLAMA_EXTRA_ARGS} ) cmd+=( "${extra_args[@]}" ) fi if (( TIMEOUT_SECONDS > 0 )) && command -v timeout >/dev/null 2>&1; then run_cmd=( timeout "$TIMEOUT_SECONDS" "${cmd[@]}" ) else run_cmd=( "${cmd[@]}" ) fi { echo "model=$model" echo "gpu_id=$gpu_id" echo "worker_slot=$slot" echo "use_gpu=$use_gpu" echo "log_file=$log_file" echo "output_file=$output_file" echo "rendered_prompt_file=$rendered_prompt_file" printf 'command=' printf '%q ' "${run_cmd[@]}" echo echo } >"$log_file" if (( use_gpu == 1 )); then if HIP_VISIBLE_DEVICES="$gpu_id" "${run_cmd[@]}" >"$output_file" 2>>"$log_file"; then status=0 else status=$? fi else if "${run_cmd[@]}" >"$output_file" 2>>"$log_file"; then status=0 else status=$? fi fi if (( status == 0 )); then generated_output="$(cat "$output_file")" lowered_output="$(printf '%s' "$generated_output" | tr '[:upper:]' '[:lower:]')" if [[ "$lowered_output" != *crazy* ]] || [[ "$lowered_output" != *fish* ]] || [[ "$lowered_output" != *swims* ]] || [[ "$lowered_output" != *quick* ]]; then status=100 { echo echo "validation_error=generated output did not contain all target words after prompt stripping" echo "normalized_output_begin" printf '%s\n' "$lowered_output" echo "normalized_output_end" } >>"$log_file" fi fi { echo "model=$model" echo "gpu_id=$gpu_id" echo "worker_slot=$slot" echo "use_gpu=$use_gpu" echo "log_file=$log_file" echo "output_file=$output_file" echo "rendered_prompt_file=$rendered_prompt_file" echo "exit_code=$status" } >"$status_file" } active_jobs=0 job_index=0 declare -a pid_queue=() for model in "${models[@]}"; do slot=$(( job_index % WORKERS )) gpu_id="${gpu_ids[$slot]}" if (( use_gpu == 1 )); then echo "queue: $model on GPU $gpu_id (worker $slot)" | tee -a "$summary_file" else echo "queue: $model on CPU (worker $slot)" | tee -a "$summary_file" fi run_one "$model" "$gpu_id" "$slot" & pid_queue+=( "$!" ) ((active_jobs += 1)) ((job_index += 1)) if (( active_jobs >= WORKERS )); then wait "${pid_queue[0]}" pid_queue=( "${pid_queue[@]:1}" ) ((active_jobs -= 1)) fi done for pid in "${pid_queue[@]}"; do wait "$pid" done echo | tee -a "$summary_file" pass_count=0 fail_count=0 declare -a failed_models=() for model in "${models[@]}"; do model_name="$(basename "$model" .gguf)" status_file="$status_dir/${model_name}.status" if [[ ! -f "$status_file" ]]; then echo "=== $model ===" | tee -a "$summary_file" echo "result: FAIL (missing status file)" | tee -a "$summary_file" echo | tee -a "$summary_file" ((fail_count += 1)) failed_models+=( "$model" ) continue fi unset model_value gpu_value worker_value mode_value log_value output_value exit_value while IFS='=' read -r key value; do case "$key" in model) model_value="$value" ;; gpu_id) gpu_value="$value" ;; worker_slot) worker_value="$value" ;; use_gpu) mode_value="$value" ;; log_file) log_value="$value" ;; output_file) output_value="$value" ;; exit_code) exit_value="$value" ;; esac done <"$status_file" echo "=== ${model_value:-$model} ===" | tee -a "$summary_file" if [[ "${mode_value:-0}" == "1" ]]; then echo "gpu: ${gpu_value:-unknown}, worker: ${worker_value:-unknown}" | tee -a "$summary_file" else echo "mode: cpu, worker: ${worker_value:-unknown}" | tee -a "$summary_file" fi echo "log: ${log_value:-unknown}" | tee -a "$summary_file" echo "output: ${output_value:-unknown}" | tee -a "$summary_file" if [[ "${exit_value:-missing}" == "0" ]]; then echo "result: PASS" | tee -a "$summary_file" ((pass_count += 1)) else echo "result: FAIL (exit ${exit_value:-missing})" | tee -a "$summary_file" ((fail_count += 1)) failed_models+=( "${model_value:-$model}" ) fi echo | tee -a "$summary_file" done echo "passed: $pass_count" | tee -a "$summary_file" echo "failed: $fail_count" | tee -a "$summary_file" if (( fail_count > 0 )); then echo "failed_models:" | tee -a "$summary_file" for failed_model in "${failed_models[@]}"; do echo " $failed_model" | tee -a "$summary_file" done fi echo "summary: $summary_file" if (( fail_count > 0 )); then exit 1 fi