08_eval_morphbench.py

"""08: TurkMorfBench morfolojik üretim değerlendirmesi; oracle vs %20-hibrit."""
import os, json, torch, torch.nn as nn, numpy as np, re
os.environ["HF_DATASETS_OFFLINE"]="1"; os.environ["HF_HUB_OFFLINE"]="1"
from transformers import AutoModelForCausalLM, AutoTokenizer
from fla.layers import GatedDeltaNet
torch.manual_seed(0)
MID="ecloudtech/Erk-14B"
KEEP=[0,9,28,29,30,31,32,33,34,35]; LIN=[i for i in range(40) if i not in KEEP]
CFG=[1,3,5,7,10,36,38,39]
tok=AutoTokenizer.from_pretrained(MID)
model=AutoModelForCausalLM.from_pretrained(MID, torch_dtype=torch.bfloat16).to("cuda").eval()
for p in model.parameters(): p.requires_grad=False
H=model.config.hidden_size
class GDNWrap(nn.Module):
    def __init__(self, g): super().__init__(); self.gdn=g
    def forward(self, hidden_states, *a, **k):
        o=self.gdn(hidden_states); y=o[0] if isinstance(o,tuple) else o; return (y,None)
wraps={}
for li in CFG:
    g=GatedDeltaNet(hidden_size=H, head_dim=128, num_heads=40, use_gate=True, use_short_conv=True, mode="chunk")
    g.load_state_dict(torch.load(f"./gdn_weights/L{li}.pt")); g=g.to("cuda").to(torch.bfloat16).eval()
    wraps[li]=GDNWrap(g).to("cuda").to(torch.bfloat16)
orig={li:model.model.layers[li].self_attn for li in LIN}
def install(hyb):
    for li in LIN: model.model.layers[li].self_attn = (wraps[li] if (hyb and li in CFG) else orig[li])
print("model+gdn yuklendi", flush=True)

# benchmark yukle (alanlar: soru/cevap/tip)
data=None
for f in ["./data/turkmorfbench.json","./turkmorfbench.json"]:
    if os.path.exists(f):
        data=json.load(open(f,encoding="utf-8")); print(f"benchmark: {f}, {len(data)} ornek", flush=True); break
# alan adlarini otomatik bul
ex0=data[0]; print("alanlar:", list(ex0.keys()), flush=True)
SORU="soru" if "soru" in ex0 else ("question" if "question" in ex0 else list(ex0.keys())[0])
CEVAP="cevap" if "cevap" in ex0 else ("answer" if "answer" in ex0 else None)
TIP="tip" if "tip" in ex0 else ("type" if "type" in ex0 else None)

def norm(s):
    s=s.strip().lower(); s=re.sub(r"[.,!?;:\"'()]","",s)
    return s.split()[0] if s.split() else s
def gen(ids, n=12):
    out=ids
    with torch.no_grad():
        for _ in range(n):
            lg=model(out, use_cache=False).logits[0,-1]
            nxt=lg.argmax().view(1,1); out=torch.cat([out,nxt],1)
            if nxt.item()==(tok.eos_token_id or -1): break
    return tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True)

def evaluate(hyb):
    install(hyb); arr=[]; d={}; t={}
    for x in data:
        typ=str(x.get(TIP,"hepsi")) if TIP else "hepsi"
        p=str(x[SORU])+" Yalnızca tek kelimeyle cevap ver, açıklama yapma."
        text=tok.apply_chat_template([{"role":"user","content":p}], tokenize=False, add_generation_prompt=True)
        ids=tok(text, return_tensors="pt").input_ids.to("cuda")
        ok=int(norm(gen(ids))==norm(str(x[CEVAP])))
        arr.append(ok); t[typ]=t.get(typ,0)+1; d[typ]=d.get(typ,0)+ok
    per={k: round(100*d.get(k,0)/t[k],1) for k in t}
    return np.array(arr), per

print("ORACLE degerlendiriliyor...", flush=True)
oa,o_per=evaluate(False); print(f"  ORACLE TurkMorfBench: %{100*oa.mean():.1f} | {o_per}", flush=True)
print("HIBRIT degerlendiriliyor...", flush=True)
ha,h_per=evaluate(True); print(f"  HIBRIT TurkMorfBench: %{100*ha.mean():.1f} | {h_per}", flush=True)
N=len(oa); rng=np.random.default_rng(0); dif=[]
for _ in range(3000):
    idx=rng.integers(0,N,N); dif.append(100*(ha[idx].mean()-oa[idx].mean()))
lo,hi=np.percentile(dif,[2.5,97.5]); dlt=100*(ha.mean()-oa.mean())
print("\n"+"="*56+"\nOZET - TurkMorfBench\n"+"="*56, flush=True)
print(f"  ORACLE %{100*oa.mean():.1f} -> HIBRIT %{100*ha.mean():.1f} | fark {dlt:+.1f} GA[{lo:+.1f},{hi:+.1f}]")
sig="sifir DISINDA (istatistiksel fark saptandi)" if (lo>0 or hi<0) else "sifir ICINDE (bu gucte istatistiksel fark saptanmadi, esitlik kaniti degil)"
print(f"  => %95 GA {sig}", flush=True)
print("BITTI", flush=True)