"""Grade a fine-tuned model against its base on rows it never trained on.

Run fine_tune_classifier.py first: it prints your ft: model and writes test.jsonl.

  python -m pip install -U openai
  export GRIDLINE_TOKEN="gla_..."
  export FT_MODEL="ft:qwen3.5-9b:..."
  python evaluate_fine_tune.py

It creates two eval runs. Each one is a batch, charged to your prepaid credits.
"""

import json
import os
import time

from openai import OpenAI

client = OpenAI(
    base_url="https://api.gridlinegpu.com/openai/v1",
    api_key=os.environ["GRIDLINE_TOKEN"],
    timeout=60,
)

TUNED = os.environ["FT_MODEL"]
BASE = TUNED.split(":")[1]  # ft:<base>:<org>:<job>

# --- 1. Load the held-out rows ---
# Chat rows in the fine-tuning format: system, user, then the answer you expect.

rows = [json.loads(line)["messages"] for line in open("test.jsonl")]
system = rows[0][0]["content"]
items = [
    {"item": {"ticket": user["content"], "queue": answer["content"]}}
    for _, user, answer in rows
]

# --- 2. Say how an answer is graded ---
# A row passes when the model's answer is exactly the expected queue.

routing_eval = client.evals.create(
    name="ticket-routing",
    data_source_config={
        "type": "custom",
        "item_schema": {
            "type": "object",
            "properties": {"ticket": {"type": "string"}, "queue": {"type": "string"}},
        },
        "include_sample_schema": True,
    },
    testing_criteria=[
        {
            "type": "string_check",
            "name": "exact queue",
            "input": "{{sample.output_text}}",
            "reference": "{{item.queue}}",
            "operation": "eq",
        }
    ],
)


def run_request(model):
    return {
        "name": model,
        "data_source": {
            "type": "completions",
            "model": model,
            "input_messages": {
                "type": "template",
                "template": [
                    {"role": "system", "content": system},
                    {"role": "user", "content": "{{item.ticket}}"},
                ],
            },
            "sampling_params": {"temperature": 0, "max_completion_tokens": 8},
            "source": {"type": "file_content", "content": items},
        },
    }


# --- 3. Ask what it can cost ---
# dry_run answers the most a run can charge and creates nothing.

quote = client.post(
    f"/evals/{routing_eval.id}/runs",
    body={**run_request(TUNED), "dry_run": True},
    cast_to=object,
)
print("each run costs at most $", quote["quote"]["max_charge_usd"])

# --- 4. Run it on both models ---

runs = [client.evals.runs.create(routing_eval.id, **run_request(model)) for model in (BASE, TUNED)]
while any(run.status in ("queued", "in_progress") for run in runs):
    time.sleep(30)
    runs = [client.evals.runs.retrieve(run.id, eval_id=routing_eval.id) for run in runs]

# --- 5. Compare ---

for run in runs:
    counts = run.result_counts
    print(f"{run.name}: {run.status}, {counts.passed} of {counts.total} passed")

failed = client.evals.runs.output_items.list(runs[1].id, eval_id=routing_eval.id, status="fail")
for item in failed:
    print("missed:", item.datasource_item["ticket"])
