"""Error-message feedback experiment (jokeuncle.pages.dev, 2026-11-17).

Each task in tasks.py: the agent already made one tool call that broke a constraint the tool spec
does not mention. Only the error text varies (vague / located / native); we grade the retry.

  python3 run.py qwen            # local llama-server on :18972, temperature 0
  python3 run.py sonnet          # via `claude -p` (Claude Code CLI), tools disabled
  python3 run.py qwen --no-exit  # same prompt without the "give_up" escape hatch
"""
import json, os, re, subprocess, sys, urllib.request
from concurrent.futures import ThreadPoolExecutor
from tasks import TASKS, errors

EXIT_LINE = '\nIf you cannot fix the call, reply {"give_up": "<what you would ask the user>"}.'

def prompt(t, err, with_exit=True):
    p = f"""You are an agent that operates software by calling tools. Reply with ONLY one JSON object, no prose.

Tool available: {t['spec']}
User request: {t['request']}

Your previous call:
{json.dumps({"tool": t['tool'], "args": t['bad']})}
Tool response:
{json.dumps(err)}

Make your next call as {{"tool": "...", "args": {{...}}}}.""" + EXIT_LINE
    return p if with_exit else p.replace(EXIT_LINE, "")

def qwen(p):
    body = {"messages": [{"role": "user", "content": p}], "temperature": 0, "max_tokens": 200, "seed": 0}
    req = urllib.request.Request("http://127.0.0.1:18972/v1/chat/completions",
                                 json.dumps(body).encode(), {"Content-Type": "application/json"})
    return json.load(urllib.request.urlopen(req, timeout=120))["choices"][0]["message"]["content"]

def sonnet(p):
    # CLI latency varied from 3 s to minutes in my runs, so: timeout + retry.
    for _ in range(4):
        try:
            r = subprocess.run(["claude", "-p", "--model", "sonnet", "--tools", "", "--strict-mcp-config"],
                               input=p, capture_output=True, text=True, timeout=90)
            if r.stdout.strip():
                return r.stdout
        except subprocess.TimeoutExpired:
            pass
    return ""

def parse(s):
    m = re.search(r"\{.*\}", s, re.S)
    if not m: return None
    try: return json.loads(m.group(0))
    except Exception: return None

def grade(t, raw):
    j = parse(raw)
    if j is None: return "parse_error"
    if "give_up" in j: return "give_up"
    args = j.get("args") if isinstance(j.get("args"), dict) else {}
    if t["check"](args): return "fixed"
    return "repeat" if args == t["bad"] else "wrong_fix"

if __name__ == "__main__":
    model = sys.argv[1]
    with_exit = "--no-exit" not in sys.argv
    call = {"qwen": qwen, "sonnet": sonnet}[model]
    fn = f"results_{model}{'' if with_exit else '_nogiveup'}.jsonl"
    done = {(json.loads(l)["task"], json.loads(l)["arm"]) for l in open(fn)} if os.path.exists(fn) else set()
    jobs = [(t, arm, err) for t in TASKS for arm, err in errors(t).items() if (t["id"], arm) not in done]

    def work(job):
        t, arm, err = job
        raw = call(prompt(t, err, with_exit))
        return dict(model=model, task=t["id"], kind=t["kind"], arm=arm, status=grade(t, raw), raw=raw.strip())

    with open(fn, "a") as out, ThreadPoolExecutor(4 if model == "sonnet" else 1) as ex:
        for rec in ex.map(work, jobs):
            out.write(json.dumps(rec, ensure_ascii=False) + "\n"); out.flush()
            print(rec["task"], rec["arm"], rec["status"], flush=True)
