How to Run NVIDIA Dory 4B with llama.cpp: Loop by Loop Guide

In this tutorial, we work with Dory 1.0 4B , a small reasoning model from NVIDIA’s Nemotron Labs whose middle layers form a shared block that can be applied once, twice, three, or four times for every token. Each extra pass, or loop, spends more computation per token without adding a single parameter, so one checkpoint can run at four different compute budgets with no retraining. We build the llama.cpp branch that NVIDIA’s model card pins, download the 3.7 GB GGUF, and read the architecture straight from the file to predict what each loop should cost in parameters and cache memory. We then check those predictions against the running server, measure decode speed at every loop count, test whether the cheaper last-loop cache changes the model’s predictions, and run a paired GSM8K evaluation to see how accuracy, answer length, and time per solved question move as the loops increase. Copy Code Copied Use a different Browser import os import re import sys import json import time import math import random import signal import shutil import hashlib import platform import statistics import traceback import subprocess import collections import urllib.request from pathlib import Path RESULTS = {} # step -> one-line result, for the summary table CLAIMS = [] # (step, claim, held) CURRENT = ["1"] def banner(title): print("\n" + "=" * 78) print(title) print("=" * 78) def section(name): def wrap(fn): def run(*a, **kw): CURRENT[0] = name.split(".")[0] banner(name) t0 = time.time() try: out = fn(*a, **kw) RESULTS[name] = out if isinstance(out, str) else "ok" return out except Exception as e: RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}" print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}") traceback.print_exc(limit=3) return None finally: print(f"\n[{time.time() - t0:.1f} s]") return run return wrap def claim(text, held): CLAIMS.append((CURRENT[0], text, bool(held))) return held banner("1. Build the pinned Dory llama.cpp fork and record the environment") FORK = "https://github.com/brianchmielNV/llama.cpp.git" FORK_COMMIT = "98a74c97ea09ac72f6ceda1559808fc0a2f447ce" # the commit NVIDIA's model card pins; ggml-org PR #30306 is still a draft GGUF_REPO = "nvidia/NVIDIA-Nemotron-Labs-Dory1.0-4B-GGUF" GGUF_REV = "df53755cb8741843af8e019e088819fbef17e29b" GGUF_FILE = "NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf" # the card's commands call it dory-q5_k_m.gguf GGUF_SIZE = 3_673_288_864 GGUF_SHA256 = "96080be49ffd67b916018b7bdef8c484762576fb52780e10ab98ac4978a660e0" GSM8K_REV = "740312add88f781978c0658806c59bc2815b9866" WORK = Path(os.environ.get("DORY_WORK") or ("/content/dory" if os.path.isdir("/content") else "dory_work")).resolve() WORK.mkdir(parents=True, exist_ok=True) (WORK / "logs").mkdir(exist_ok=True) subprocess.run([sys.executable, "-m", "pip", "install", "-q", "gguf==0.19.0", "huggingface_hub", "pandas", "pyarrow"], check=True) SYSTEM = platform.system() GPU = "" if shutil.which("nvidia-smi"): q = subprocess.run(["nvidia-smi", "--query-gpu=name,memory.total,compute_cap", "--format=csv,noheader"], capture_output=True, text=True) GPU = q.stdout.strip().splitlines()[0] if q.returncode == 0 and q.stdout.strip() else "" BACKEND = "metal" if SYSTEM == "Darwin" else ("cuda" if GPU else "cpu") SRC = WORK / "llama.cpp-dory" SERVER_BIN = SRC / "build" / "bin" / "llama-server" free_gb = shutil.disk_usage(WORK).free / 1e9 print(f"{SYSTEM} {platform.machine()}, Python {platform.python_version()}, {os.cpu_count()} CPU cores, " f"backend {BACKEND}{' (' + GPU + ')' if GPU else ''}, {free_gb:.1f} GB free disk") if not SERVER_BIN.exists(): t0 = time.time() if not (SRC / ".git").exists(): subprocess.run(["git", "init", "-q", str(SRC)], check=True) subprocess.run(["git", "-C", str(SRC), "remote", "add", "origin", FORK], check=True) subprocess.run(["git", "-C", str(SRC), "fetch", "-q", "--depth", "1", "origin", FORK_COMMIT], check=True) subprocess.run(["git", "-C", str(SRC), "checkout", "-q", "--detach", "FETCH_HEAD"], check=True) flags = ["-DCMAKE_BUILD_TYPE=Release", "-DLLAMA_BUILD_SERVER=ON", "-DLLAMA_BUILD_TESTS=OFF", "-DLLAMA_BUILD_EXAMPLES=OFF", "-DLLAMA_BUILD_APP=OFF", "-DLLAMA_BUILD_UI=OFF", "-DLLAMA_USE_PREBUILT_UI=ON", "-DLLAMA_OPENSSL=OFF"] if BACKEND == "metal": # the card's Apple-silicon flags flags += ["-DCMAKE_OSX_ARCHITECTURES=arm64", "-DGGML_METAL=ON", "-DGGML_METAL_EMBED_LIBRARY=ON", "-DGGML_BLAS=ON", "-DGGML_BLAS_VENDOR=Apple"] elif BACKEND == "cuda": # kernels for this GPU only (a T4 is 7.5); flash-attention kernels are unused with --flash-attn off cap = GPU.split(",")[-1].strip().replace(".", "") flags += ["-DGGML_CUDA=ON", f"-DCMAKE_CUDA_ARCHITECTURES={cap}", "-DGGML_CUDA_FA=OFF", "-DGGML_NATIVE=OFF"] env = dict(os.environ, HF_UI_VERSION="b11551") with open(WORK / "build.log", "w") as log: subprocess.run(["cmake", "-S", str(SRC), "-B", str(SRC / "build")] + flags, check=True, env=env, stdout=log, stderr=log) subprocess.run(["cmake", "--build", str(SRC / "build"), "--parallel", str(os.cpu_count()), "--target", "llama-server"], check=True, env=env, stdout=log, stderr=log) print(f"built llama-server in {time.time() - t0:.0f} s") head = subprocess.run(["git", "-C", str(SRC), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip() assert head == FORK_COMMIT, f"checked out {head}, expected {FORK_COMMIT}" version = subprocess.run([str(SERVER_BIN), "--version"], capture_output=True, text=True) print(f"fork commit {head[:12]}; {(version.stdout + version.stderr).strip().splitlines()[-1]}") RESULTS["1. Build the pinned Dory llama.cpp fork and record the environment"] = f"fork {head[:12]}, backend {BACKEND}" We build llama-server from the exact commit of the Dory branch that NVIDIA’s model card pins, because Dory support in llama.cpp is still an open draft pull request and no released binary can load the model. The cell detects the platform and passes the card’s Metal flags on a Mac, or compiles CUDA kernels for the attached GPU alone on Colab, skipping the flash-attention kernels that the card’s –flash-attn off never uses, which keeps a T4 build manageable. It then checks that the checked-out commit is the pinned one, so every later number traces to one known version of the code. Copy Code Copied Use a different Browser @section("2. Download the Q5_K_M GGUF at a pinned revision and check its SHA-256") def download_step(): global GGUF from huggingface_hub import hf_hub_download t0 = time.time() GGUF = Path(hf_hub_download(GGUF_REPO, GGUF_FILE, revision=GGUF_REV, local_dir=str(WORK))) h = hashlib.sha256() with open(GGUF, "rb") as f: for block in iter(lambda: f.read(1 The Q5_K_M GGUF comes from Hugging Face at a fixed repository revision, and we hash it before anything else touches it. The card’s commands refer to the file as dory-q5_k_m.gguf, but the repository names it NVIDIA-Nemotron-Labs-Dory1.0-4B-Q5_K_M.gguf, so copying the card’s command verbatim fails; pinning the real name, the size, and the SHA-256 together catches both a wrong file and a corrupted download. The 3.67 GB file matches the card’s published checksum exactly, so every measurement that follows uses the same bytes NVIDIA published. Copy Code Copied Use a different Browser @section("3. Read the architecture from the GGUF: three layer groups and a default of four loops") def architecture_step(): global META, GROUPS, PATTERN import gguf reader = gguf.GGUFReader(str(GGUF)) META = {k: f.contents() for k, f in reader.fields.items() if k.startswith("dory.")} for k in ("dory.block_count", "dory.input_layer_count", "dory.recurrent_layer_count", "dory.output_layer_count", "dory.num_loops", "dory.recurrent_kv_cache_mode", "dory.attention.sliding_window", "dory.context_length"): print(f"{k:34s} {META[k]}") PATTERN = META["dory.layer_pattern"].split("-")[:-1] # one symbol per attention layer: S sliding window, * full n_in, n_rec = META["dory.input_layer_count"], META["dory.recurrent_layer_count"] print(f"\nlayer entries alternate attention and MLP: {len(PATTERN)} attention layers