Python usage

Run audits from notebooks, training scripts, and pipelines.

Minimal audit

from modeldiffr import run_audit
from modeldiffr.backend import HFBackend
from modeldiffr.report import write_reports
from modeldiffr.suites import get_suite

base = HFBackend("Qwen/Qwen2.5-0.5B-Instruct")
derivative = HFBackend("your-org/your-qwen-finetune")
result = run_audit(base, derivative, get_suite("quick"))
write_reports(result, "audit")
print(result["verdict"])

With progress and pinned settings

from modeldiffr import run_audit
from modeldiffr.backend import HFBackend
from modeldiffr.report import write_reports
from modeldiffr.suites import get_suite

base = HFBackend(
    "Qwen/Qwen2.5-0.5B-Instruct",
    device="cuda:0",
    revision="YOUR_COMMIT_SHA",
    dtype="bfloat16",
)
derivative = HFBackend("./checkpoints/step_4000", device="cuda:0", dtype="bfloat16")

result = run_audit(
    base,
    derivative,
    get_suite("quick"),
    seed=0,
    n_resamples=5000,
    progress=print,
)
json_path, html_path = write_reports(result, "audit/step_4000")

for metric in result["metrics"]:
    status = "CHANGED" if metric["significant"] else "within noise"
    print(f"{metric['name']}: delta {metric['delta']:+.4f} ({status})")
print(f"open {html_path}")

Audit every checkpoint in a training run

Load the base once and reuse it.

from pathlib import Path

from modeldiffr import run_audit
from modeldiffr.backend import HFBackend
from modeldiffr.report import write_reports
from modeldiffr.suites import get_suite

suite = get_suite("quick")
base = HFBackend("Qwen/Qwen2.5-0.5B-Instruct", dtype="bfloat16")

for ckpt in sorted(Path("checkpoints").glob("step_*")):
    derivative = HFBackend(str(ckpt), dtype="bfloat16")
    result = run_audit(base, derivative, suite)
    write_reports(result, f"audits/{ckpt.name}")
    print(ckpt.name, result["verdict"]["changed"])
    del derivative

Statistics without models

paired_bootstrap is exported from the top level and works on any paired scores.

from modeldiffr import paired_bootstrap

base = [1, 0, 1, 1, 0, 1, 1, 1]
derivative = [1, 1, 1, 1, 1, 1, 1, 1]
print(paired_bootstrap(base, derivative, seed=0))

See the API reference for every parameter.