[HSC-426] 동일 조건 비교와 키워드 채점 추가

experiments/consultation-agent/.gitignore

@@ -3,3 +3,4 @@ __pycache__/
 runs/
 *.sqlite*
 .env
+inputs/

experiments/consultation-agent/README.md

@@ -29,3 +29,24 @@ python3 -m venv .venv
 ## 적용 범위
 
 독립 실행하는 후속 실험 코드다. FastAPI 서비스와 연결한 운영 기능은 아니다. 응답을 받았는지 확인할 수 없는 중단은 `STOPPED_UNCERTAIN`으로 남기고 자동 재호출하지 않는다. 신고, 고객 점수 변경, 블랙리스트 처리는 구현하지 않았다.
+
+## 비교 실행과 채점
+
+`evaluation.py`는 짝이 빠지거나 최초 입력·응답이 달라진 비교를 거부한다. `keyword_score.py`는 사람 정답과 키워드 집합을 대조한다. 정답 파일은 모델 입력에 포함하지 않는다.
+
+`evaluate.py`, `replay_catalog.py`, `run_human50.py`는 기존 실험의 실행 도구다. 유료 실행에는 `OPENAI_API_KEY` 환경변수가 필요하다. 개인 PC의 키 파일을 읽지 않는다. HTTP 클라이언트는 `experiment.py`에 있다.
+
+새 개발 실행은 `inputs/`에 원래 형식의 입력 자료를 준비한 뒤 생성한다. 상담 원문·정답·응답 기록·SQLite 상태 파일은 이 저장소에 포함하지 않는다.
+
+- `regression`: `semantic_regressions.json`의 `id`, `text`, `required`, `forbidden`, `sentiment`와 `business_keyword_alias_map.snapshot.json`이 필요하다.
+- `development`: `dataset_summary.json`의 `source_root`, `development_50.jsonl`의 원본 경로·레코드 번호·해시, 같은 사전 파일이 필요하다.
+
+```sh
+.venv/bin/python evaluate.py prepare --name development_local --dataset regression
+# 입력과 protocol.json을 확인하고 API 키를 환경에 설정한 뒤 실행한다.
+.venv/bin/python evaluate.py run --name development_local
+```
+
+준비 단계는 API를 호출하지 않는다. 실행은 최대 50건으로 제한한다. `run_human50.py`는 별도로 동결한 `runs/human_keywords_v1`이 있어야 하며, 원자료 없이 실행할 수 없다. `replay_catalog.py`는 완료된 `runs/development_v2`의 최초 응답 50개를 재사용한다.
+
+원본 실험의 코드 스냅샷 해시는 원본에 대한 기록이다. 경로 정리 후 코드는 해시가 달라지므로 과거 실행의 protocol을 그대로 재사용하지 않는다. 새 실행을 준비하면 현재 소스와 입력으로 새 해시가 만들어진다.

experiments/consultation-agent/evaluate.py

@@ -0,0 +1,92 @@
+"""Prepare frozen development runs and execute real shared-contract comparisons."""
+import argparse,asyncio,hashlib,importlib.metadata,json,re,sys,time
+from pathlib import Path
+import contract as c
+from evaluation import ARMS,summarize
+from journal import Journal
+from mcp_transport import MCPTools
+from runner import Limits,run_case
+
+HERE=Path(__file__).resolve().parent;P=HERE/'inputs'
+SOURCES=['contract.py','journal.py','runner.py','evaluation.py','evaluate.py','tool_backend.py','mcp_server.py','mcp_transport.py']
+def source_map():return {**{name:HERE/name for name in SOURCES},'experiment_client.py':HERE/'experiment.py'}
+def sha(data):return hashlib.sha256(data).hexdigest()
+def dump(path,value):path.write_text(json.dumps(value,ensure_ascii=False,indent=2))
+def folder(name):
+    if not re.fullmatch('[a-z0-9_-]{1,60}',name):raise ValueError('invalid_run_name')
+    return HERE/'runs'/name
+def prepare(name,dataset):
+    out=folder(name);out.mkdir(parents=True,exist_ok=False)
+    expected=None;rows=[]
+    if dataset=='regression':
+        source=json.loads((P/'semantic_regressions.json').read_text())
+        rows=[{'id':r['id'],'text':r['text']} for r in source]
+        expected={r['id']:{k:r[k] for k in ['required','forbidden','sentiment']} for r in source}
+    elif dataset=='development':
+        root=Path(json.loads((P/'dataset_summary.json').read_text())['source_root'])
+        for line in (P/'development_50.jsonl').read_text().splitlines():
+            r=json.loads(line);data=(root/r['path']).read_bytes()
+            if sha(data)!=r['source_file_sha256']:raise ValueError('source_hash_changed')
+            text=json.loads(data)[r['record_index']]['consulting_content']
+            text=re.sub(r'[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}','<EMAIL>',text)
+            text=re.sub(r'(?<!\d)(?:01[016789]|0[2-6][1-5]?)[ -]?\d{3,4}[ -]?\d{4}(?!\d)','<PHONE_NUMBER>',text)
+            rows.append({'id':r['id'],'text':text,'source_file_sha256':r['source_file_sha256']})
+    else:raise ValueError('Only pre-existing development runs allowed; 2000 gated separately')
+    for r in rows:r['input_sha256']=sha(r['text'].encode())
+    dump(out/'manifest.json',rows)
+    (out/'taxonomy.json').write_bytes((P/'business_keyword_alias_map.snapshot.json').read_bytes())
+    if expected:dump(out/'expected_development_constraints.json',expected)
+    for name,path in source_map().items():(out/name).write_bytes(path.read_bytes())
+    files=list(source_map())+['manifest.json','taxonomy.json']+(['expected_development_constraints.json'] if expected else [])
+    dump(out/'protocol.json',{'dataset':dataset,'cases':len(rows),'model':c.MODEL,'arms':ARMS,'model_call_ceiling':4,'tool_call_ceiling':6,'run_budget_usd':2.0,
+        'human_gold':0,'initial_response':'shared','arm_order':'cyclic latin rotation','information':'all complete turns and full dictionary preloaded identically',
+        'constraints':'same strict customer-turn-ID schema, same reconstruction/dedup/validation',
+        'sdk':importlib.metadata.version('mcp'),'hashes':{name:sha((out/name).read_bytes()) for name in files},
+        'scope':'development debug only, not held-out semantic accuracy or 2000-case evaluation'})
+    print(json.dumps({'prepared':str(out),'cases':len(rows)}))
+async def run(name):
+    out=folder(name)
+    if (out/'completion.json').exists():
+        print('Already complete; no mutation or API');return
+    protocol=json.loads((out/'protocol.json').read_text())
+    if protocol['cases']>50:raise ValueError('Large evaluation gate not passed')
+    for name,h in protocol['hashes'].items():
+        if sha((out/name).read_bytes())!=h:raise ValueError('snapshot_changed')
+        if name in source_map() and sha(source_map()[name].read_bytes())!=h:raise ValueError('source_changed_after_freeze')
+    if importlib.metadata.version('mcp')!=protocol['sdk']:raise ValueError('sdk_changed')
+    import experiment as e
+    api=e.client_from_env()
+    retrylog=(out/'retry_events.jsonl').open('a')
+    async def model(payload):
+        for attempt in range(5):
+            try:return await asyncio.to_thread(api,payload)
+            except e.APIError as ex:
+                retrylog.write(json.dumps({'code':str(ex),'attempt':attempt+1,'payload_hash':c.digest(payload)})+'\n');retrylog.flush()
+                if 'rate_limit_exceeded' not in str(ex) or attempt==4:raise
+                await asyncio.sleep(20)
+    journal=Journal(out/'state.sqlite');rows=json.loads((out/'manifest.json').read_text());tax=json.loads((out/'taxonomy.json').read_text())
+    results=[];started=time.perf_counter();startup=time.perf_counter()
+    try:
+        async with MCPTools(out/'manifest.json',out/'taxonomy.json') as tools:
+            dump(out/'transport_startup.json',{'seconds':time.perf_counter()-startup,'sdk':protocol['sdk']})
+            for index,row in enumerate(rows):
+                order=ARMS[index%4:]+ARMS[:index%4]
+                for arm in order:
+                    result=await run_case(arm,row,tax,model,tools,journal,Limits())
+                    results.append(result)
+                    dump(out/'progress.json',{'completed_arm_results':len(results),'cases_total':len(rows),'actual_receipt_estimated_usd':journal.total_cost(),'elapsed_seconds':time.perf_counter()-started})
+                    print(json.dumps({'id':row['id'],'arm':arm,'status':result['status'],'model_calls':result['model_calls'],'tool_calls':result['tool_calls'],'actual_usd':round(journal.total_cost(),8)}),flush=True)
+        (out/'results.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in results))
+        receipts=journal.receipts();dump(out/'receipts.json',receipts)
+        # Gold-like development expectations are loaded ONLY after all model execution.
+        path=out/'expected_development_constraints.json';expected=json.loads(path.read_text()) if path.exists() else None
+        summary=summarize(results,receipts,[r['id'] for r in rows],expected)
+        dump(out/'summary.json',summary);dump(out/'completion.json',{'cases':len(rows),'arm_results':len(results),'actual_usd':journal.total_cost(),'elapsed_seconds':time.perf_counter()-started})
+        print(json.dumps(summary,ensure_ascii=False),flush=True)
+    finally:
+        dump(out/'receipts.json',journal.receipts());journal.close();retrylog.close()
+if __name__=='__main__':
+    parser=argparse.ArgumentParser();parser.add_argument('action',choices=['prepare','run']);parser.add_argument('--name',required=True);parser.add_argument('--dataset',choices=['regression','development'])
+    args=parser.parse_args()
+    if args.action=='prepare':prepare(args.name,args.dataset)
+    else:asyncio.run(run(args.name))

experiments/consultation-agent/evaluation.py

@@ -0,0 +1,39 @@
+"""Paired operational metrics; never turn structural validity into accuracy."""
+import collections
+import statistics
+
+ARMS=['single','fixed','budget_review','adaptive']
+
+def regression_checks(row,expected):
+    pred=row.get('prediction') or {}
+    names={k['name'] for k in pred.get('keywords',[])}
+    missing=sorted(set(expected['required'])-names)
+    forbidden=sorted(set(expected['forbidden'])&names)
+    sentiment_match=pred.get('sentiment')==expected['sentiment']
+    return {'missing':missing,'forbidden':forbidden,'sentiment_match':sentiment_match,'pass':not missing and not forbidden and sentiment_match,'kind':'authored_development_constraints_not_human_gold'}
+
+def summarize(rows,receipts,case_ids,expected=None):
+    if len(case_ids)!=len(set(case_ids)):raise ValueError('duplicate_case_ids')
+    pairs={(r['id'],r['arm']):r for r in rows}
+    if len(pairs)!=len(rows) or set(pairs)!={(i,a) for i in case_ids for a in ARMS}:raise ValueError('missing_duplicate_or_extra_pairs')
+    for i in case_ids:
+        group=[pairs[(i,a)] for a in ARMS]
+        if len({r['base_payload_hash'] for r in group})!=1:raise ValueError('nonidentical_base_payload')
+        if any(r['base_raw']!=group[0]['base_raw'] for r in group):raise ValueError('nonidentical_initial_response')
+    output={'cases':len(case_ids),'arms':{},'human_gold_verified':0,'actual_success_receipt_cost_usd':sum(r['cost'] or 0 for r in receipts if r['status']=='COMPLETE'),
+      'actual_success_calls':sum(r['status']=='COMPLETE' for r in receipts),'uncertain_calls':sum(r['status']!='COMPLETE' for r in receipts),
+      'fairness':{'same_initial_payload_and_response':True,'shared_final_contract':True,'complete_input_and_dictionary_all_arms':True,'equal_max_model_calls_for_budget_review_and_adaptive':True,'same_actual_token_budget':False},
+      'limitations':['Shared initial completion design; logical arm costs include the reused base receipt','Request seconds include historical shared receipt latency; current elapsed excludes MCP session startup','No human semantic gold; authored regression checks are development-only','Equal call ceilings are not equal realized token spend'],
+      'usefulness_gate':{'passed':False,'status':'pending_external_evidence_review','reason':'Structural metrics and authored development constraints alone do not establish useful adaptive-agent superiority or semantic accuracy.'}}
+    for arm in ARMS:
+        group=[pairs[(i,arm)] for i in case_ids]
+        output['arms'][arm]={'structural_valid':sum(r['prediction'] is not None for r in group),'validated_status':sum(r['status']=='VALIDATED' for r in group),
+          'review_required':sum(r['status']=='REVIEW_REQUIRED' for r in group),'estimated_usd':sum(r['estimated_usd'] for r in group),'uncached_usd':sum(r['uncached_usd'] for r in group),
+          'logical_model_calls':sum(r['model_calls'] for r in group),'tool_calls':sum(r['tool_calls'] for r in group),'tool_errors':sum(r['tool_errors'] for r in group),
+          'mean_logical_request_seconds':statistics.mean(r['request_seconds'] for r in group),'mean_current_elapsed_seconds':statistics.mean(r['elapsed_seconds'] for r in group),
+          'tool_seconds':sum(t.get('seconds',0) for r in group for t in r['trace'] if t['type']=='tool'),
+          'stop_reasons':dict(collections.Counter(r.get('stop_reason','fixture') for r in group))}
+        if expected:
+            scores=[regression_checks(pairs[(i,arm)],expected[i]) for i in case_ids]
+            output['arms'][arm]['authored_development_constraints_pass']=sum(x['pass'] for x in scores)
+    return output

experiments/consultation-agent/experiment.py

@@ -0,0 +1,40 @@
+"""HTTP client retained from the local comparison experiment."""
+import json
+import os
+import ssl
+import urllib.error
+import urllib.request
+
+class APIError(Exception):
+    pass
+
+def client(key):
+    try:
+        import certifi
+        context = ssl.create_default_context(cafile=certifi.where())
+    except ImportError:
+        context = ssl.create_default_context()
+    def call(payload):
+        req = urllib.request.Request('https://api.openai.com/v1/chat/completions',
+            data=json.dumps(payload, ensure_ascii=False).encode(),
+            headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'})
+        try:
+            with urllib.request.urlopen(req, timeout=60, context=context) as response:
+                return json.load(response)
+        except urllib.error.HTTPError as exc:
+            try:
+                code = json.loads(exc.read()).get('error', {}).get('code')
+            except Exception:
+                code = 'unparsed'
+            # Never print response bodies, request headers or credentials.
+            raise APIError(f'HTTP {exc.code}; code={code}') from None
+        except urllib.error.URLError:
+            raise APIError('NETWORK_ERROR') from None
+    return call
+
+
+def client_from_env():
+    key = os.environ.get("OPENAI_API_KEY", "").strip()
+    if not key:
+        raise ValueError("OPENAI_API_KEY is required")
+    return client(key)

experiments/consultation-agent/keyword_score.py

@@ -0,0 +1,17 @@
+"""Keyword-only scoring against supplied labels. Sentiment intentionally excluded."""
+def score(rows,gold):
+    if len(rows)!=len(gold) or len({r['id'] for r in rows})!=len(rows) or {r['id'] for r in rows}!=set(gold):
+        raise ValueError('incomplete_or_duplicate_pairs')
+    tp=fp=fn=exact=review=0
+    details=[]
+    for r in rows:
+        expected=set(gold[r['id']]);pred=r.get('prediction')
+        names={x['name'] for x in (pred or {}).get('keywords',[])}
+        held=r['status']!='VALIDATED' or pred is None
+        # All-case quality counts abstention as no delivered labels, never free success.
+        delivered=set() if held else names
+        a,b,c=len(delivered&expected),len(delivered-expected),len(expected-delivered)
+        tp+=a;fp+=b;fn+=c;review+=held
+        ok=not held and delivered==expected;exact+=ok
+        details.append({'id':r['id'],'exact':ok,'review_required':held,'missing':sorted(expected-delivered),'extra':sorted(delivered-expected),'raw_predicted_keywords':sorted(names)})
+    return {'cases':len(rows),'tp':tp,'fp':fp,'fn':fn,'micro_precision':tp/(tp+fp) if tp+fp else None,'micro_recall':tp/(tp+fn) if tp+fn else None,'micro_f1':2*tp/(2*tp+fp+fn) if 2*tp+fp+fn else None,'exact_matches':exact,'review_required':review,'details':details}