Structured output and tool calling

The format parameter for JSON and JSON schemas, grammar-constrained decoding, tools in /api/chat, and validating what comes back.

Constrained JSON output

import json
import requests

BASE = "http://localhost:11434"

# 1. ask for JSON: no structural guarantee, the model merely complies
loose = requests.post(f"{BASE}/api/generate", json={
    "model": "llama3.2:3b-instruct-q4_K_M",
    "prompt": "Give the capital of France and its population as JSON.",
    "format": "json",
    "stream": False,
    "options": {"temperature": 0.0},
}, timeout=120).json()["response"]
print(json.loads(loose))

# 2. give a JSON schema: the sampler is constrained to produce a match
SCHEMA = {
    "type": "object",
    "properties": {
        "category": {"type": "string",
                     "enum": ["billing", "technical", "account", "other"]},
        "urgency": {"type": "string", "enum": ["low", "normal", "high"]},
        "order_id": {"type": ["string", "null"]},
        "summary": {"type": "string"},
    },
    "required": ["category", "urgency", "summary"],
}

def classify(text, model="llama3.2:3b-instruct-q4_K_M"):
    response = requests.post(f"{BASE}/api/chat", json={
        "model": model,
        "messages": [
            {"role": "system",
             "content": "Classify the support message. Return only the object."},
            {"role": "user", "content": text},
        ],
        "format": SCHEMA,
        "stream": False,
        "options": {"temperature": 0.0},
    }, timeout=120)
    response.raise_for_status()
    return json.loads(response.json()["message"]["content"])

print(classify("I was charged twice for order A-1042 and cannot log in."))

• format: "json" asks for syntactically valid JSON. format: <schema> constrains decoding to the schema, so enums and required fields are enforced by the sampler, not by luck.
• Constrained decoding needs a capable model to be useful: the output is guaranteed well-formed but can still be semantically wrong.
• Schema support varies by model. Vision models, some very small models and some quantisations handle grammars poorly; test before relying on it.
• Keep the schema small. Every property and enum value is grammar the sampler must satisfy, and a large schema measurably slows generation.

Tool calling in /api/chat

import json
import requests

BASE = "http://localhost:11434"

TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "get_order_status",
            "description": "Look up the delivery status of an order by its id.",
            "parameters": {
                "type": "object",
                "properties": {"order_id": {"type": "string",
                                            "description": "Order id such as A-1042"}},
                "required": ["order_id"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "issue_refund",
            "description": "Refund an order. Requires the order id and a reason.",
            "parameters": {
                "type": "object",
                "properties": {"order_id": {"type": "string"},
                               "reason": {"type": "string"}},
                "required": ["order_id", "reason"],
            },
        },
    },
]

def get_order_status(order_id):
    return {"order_id": order_id, "status": "in_transit", "eta": "2026-09-21"}

HANDLERS = {"get_order_status": get_order_status,
            "issue_refund": lambda **kw: {"error": "not permitted in this demo"}}

messages = [{"role": "user", "content": "Where is order A-1042?"}]

for turn in range(4):                      # bounded loop, always
    response = requests.post(f"{BASE}/api/chat", json={
        "model": "qwen2.5:7b-instruct-q4_K_M",
        "messages": messages,
        "tools": TOOLS,
        "stream": False,
        "options": {"temperature": 0.0},
    }, timeout=180).json()

    message = response["message"]
    messages.append(message)

    calls = message.get("tool_calls") or []
    if not calls:
        print("final:", message["content"])
        break

    for call in calls:
        name = call["function"]["name"]
        args = call["function"]["arguments"]
        if isinstance(args, str):
            args = json.loads(args)
        result = HANDLERS[name](**args) if name in HANDLERS else {"error": "unknown tool"}
        messages.append({"role": "tool", "name": name,
                         "content": json.dumps(result)})
else:
    print("stopped: tool loop exceeded the turn limit")

Validation and repair

from pydantic import BaseModel, ValidationError
from typing import Literal, Optional

class Ticket(BaseModel):
    category: Literal["billing", "technical", "account", "other"]
    urgency: Literal["low", "normal", "high"]
    order_id: Optional[str] = None
    summary: str

def safe_classify(text, attempts=2):
    last_error = None
    for attempt in range(attempts):
        raw = classify(text)
        try:
            return {"ok": True, "ticket": Ticket.model_validate(raw)}
        except ValidationError as exc:
            last_error = exc
            # feed the problem back rather than regenerating blindly
            text = (text + f"\n\nPrevious output was invalid: {exc.errors()[0]['msg']}. "
                            f"Return a corrected object.")
    return {"ok": False, "error": str(last_error)[:200], "input": text[:200]}

print(safe_classify("Double charge on order A-1042."))
print(safe_classify("Your app is broken!"))        # no order id present

# a repair for the common case where a small model emits a string instead of an object
def coerce(raw):
    if isinstance(raw, str):
        raw = json.loads(raw)
    if isinstance(raw.get("order_id"), dict):
        raw["order_id"] = raw["order_id"].get("value")
    return raw

• Validate every structured response before it reaches business logic. Grammar-constrained decoding guarantees syntax, not semantics or field meaning.
• Retry once with the validation error included. A second blind attempt rarely helps; a second attempt that names the failing field often does.
• Log the raw response alongside the validation error. Without it you cannot tell whether the schema is wrong or the model is.
• Count validation failures per field and review them. A field that fails 20% of the time is a prompt or schema design problem, not a model one.

FAQ

Related

Calling Ollama from code Troubleshooting and the limits of local models

Last refreshed 2026-09-18.