Evaluación A/B: comparar Kimi K2.7-Code contra Claude con juez calibrado

# evals/swap_test.py — A/B mínimo antes de cambiar de modelo en producción. # Requiere: pip install openai anthropic, y fixtures.jsonl con {"q":"..."} por línea. import os, json from openai import OpenAI from anthropic import Anthropic kimi = OpenAI(base_url="https://api.moonshot.ai/v1", api_key=os.environ["KIMI_KEY"]) claude = Anthropic() # ANTHROPIC_API_KEY judge = OpenAI() # OPENAI_API_KEY — juez calibrado, distinto de los dos competidores JUDGE = ('Compara dos respuestas a la misma tarea y devuelve SOLO JSON: ' '{"winner":"A"|"B"|"TIE","reason":"<30 palabras>"}. ' 'Criterio: corrección > completitud > brevedad.') def ask_kimi(q): return kimi.chat.completions.create(model="kimi-k2.7-code", messages=[{"role":"user","content":q}]).choices[0].message.content def ask_claude(q): return claude.messages.create(model="claude-opus-4-8", max_tokens=2048, messages=[{"role":"user","content":q}]).content[0].text for line in open("fixtures.jsonl"): q = json.loads(line)["q"] a, b = ask_kimi(q), ask_claude(q) v = judge.chat.completions.create(model="gpt-5-mini", messages=[{"role":"system","content":JUDGE}, {"role":"user","content":f"TAREA:{q}\nA:{a}\nB:{b}"}]) print(json.loads(v.choices[0].message.content))