How to Run Underdog Saluki 27B: 2-Bit Qwen3.8-27B Tool-Calling Guide

In this tutorial, we work with Underdog Saluki 27B , a 7.9 GB build of Qwen3.8-27B that its makers say keeps tool calling intact at about two bits per weight and even edges out the full 54 GB model on their own 120-task benchmark. Saluki is built on ISTA-DASLab’s GSQ-RCO quantization of the same model, so we put both files side by side in stock llama.cpp. We start with the files themselves: a tensor-by-tensor diff of the two GGUF headers, then a byte-level comparison that shows whether Saluki is a remix of ISTA’s bits or a changed model. We then serve each file on one GPU, read its memory and speed, and score both on the same seeded 120 tasks from the Berkeley Function Calling Leaderboard, using BFCL’s own checker with strict and lenient parsing. Copy Code Copied Use a different Browser import os import re import ast import sys import json import math import copy import enum import glob import time import types import random import shutil import struct import hashlib import tarfile import platform import traceback import subprocess import collections import urllib.request import urllib.error RESULTS = {} # step -> one-line result, for the summary table CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run) CURRENT = ["1"] def banner(title): print("\n" + "=" * 78) print(title) print("=" * 78) def section(name): def wrap(fn): def run(*a, **kw): CURRENT[0] = name.split(".")[0] banner(name) t0 = time.time() try: out = fn(*a, **kw) RESULTS[name] = out if isinstance(out, str) else "ok" return out except Exception as e: RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}" print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}") traceback.print_exc(limit=3) return None finally: print(f"\n[{time.time() - t0:.1f} s]") json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1) return run return wrap def claim(text, held): CLAIMS.append((CURRENT[0], text, held)) return held banner("1. Platform, pins and helpers") SYSTEM, MACHINE = platform.system(), platform.machine() ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac" WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work")) for d in ("models", "llama", "bfcl", "runs"): os.makedirs(os.path.join(WORK, d), exist_ok=True) # Everything the notebook downloads is pinned by size and SHA-256. LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2 LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844, "61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"), "Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509, "19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM] MODELS = { "Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c", file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152, sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"), "ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440", file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472, sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"), } # BFCL v4 data and BFCL's own checker code, at one gorilla commit. GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/" "berkeley-function-call-leaderboard/bfcl_eval/") BFCL_FILES = { "data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991", "data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a", "data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6", "data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84", "data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579", "data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc", "data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047", "data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0", "data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3", "eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538", "constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a", "constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd", "utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd", "model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3", } # Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published. KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"), ("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"), ("parallel_multiple", "Several different functions")] N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080 # llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler. GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0) def sh(cmd, **kw): return subprocess.run(cmd, capture_output=True, text=True, **kw) def sha256_file(path, chunk=1 << 22): h = hashlib.sha256() with open(path, "rb") as f: for b in iter(lambda: f.read(chunk), b""): h.update(b) return h.hexdigest() def fetch(url, dest, size=None, sha=None, quiet=False): """Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading.""" t0 = time.time() if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)): for attempt in range(5): r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url]) if r.returncode == 0 and (size is None or os.path.getsize(dest) == size): break time.sleep(5 * (attempt + 1)) got = os.path.getsize(dest) assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}" if sha: assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch" return time.time() - t0 print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}") if SYSTEM == "Linux": print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0]) gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip() print("gpu :", gpu) # The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels. import importlib.util if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None: subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79", "nvidia-cublas-cu12==12.9.2.10"], check=True) import nvidia.cuda_runtime, nvidia.cublas CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)] else: chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip() mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30 print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)") CUDA_LIBS = [] HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)") Everything the notebook touches is pinned before anything runs. The setup cell records the llama.cpp rel