diff --git a/experiments/consultation-agent/.gitignore b/experiments/consultation-agent/.gitignore index f2ee4a6..22079e2 100644 --- a/experiments/consultation-agent/.gitignore +++ b/experiments/consultation-agent/.gitignore @@ -3,3 +3,4 @@ __pycache__/ runs/ *.sqlite* .env +inputs/ diff --git a/experiments/consultation-agent/README.md b/experiments/consultation-agent/README.md index eefb828..038d936 100644 --- a/experiments/consultation-agent/README.md +++ b/experiments/consultation-agent/README.md @@ -29,3 +29,24 @@ python3 -m venv .venv ## 적용 범위 독립 실행하는 후속 실험 코드다. FastAPI 서비스와 연결한 운영 기능은 아니다. 응답을 받았는지 확인할 수 없는 중단은 `STOPPED_UNCERTAIN`으로 남기고 자동 재호출하지 않는다. 신고, 고객 점수 변경, 블랙리스트 처리는 구현하지 않았다. + +## 비교 실행과 채점 + +`evaluation.py`는 짝이 빠지거나 최초 입력·응답이 달라진 비교를 거부한다. `keyword_score.py`는 사람 정답과 키워드 집합을 대조한다. 정답 파일은 모델 입력에 포함하지 않는다. + +`evaluate.py`, `replay_catalog.py`, `run_human50.py`는 기존 실험의 실행 도구다. 유료 실행에는 `OPENAI_API_KEY` 환경변수가 필요하다. 개인 PC의 키 파일을 읽지 않는다. HTTP 클라이언트는 `experiment.py`에 있다. + +새 개발 실행은 `inputs/`에 원래 형식의 입력 자료를 준비한 뒤 생성한다. 상담 원문·정답·응답 기록·SQLite 상태 파일은 이 저장소에 포함하지 않는다. + +- `regression`: `semantic_regressions.json`의 `id`, `text`, `required`, `forbidden`, `sentiment`와 `business_keyword_alias_map.snapshot.json`이 필요하다. +- `development`: `dataset_summary.json`의 `source_root`, `development_50.jsonl`의 원본 경로·레코드 번호·해시, 같은 사전 파일이 필요하다. + +```sh +.venv/bin/python evaluate.py prepare --name development_local --dataset regression +# 입력과 protocol.json을 확인하고 API 키를 환경에 설정한 뒤 실행한다. +.venv/bin/python evaluate.py run --name development_local +``` + +준비 단계는 API를 호출하지 않는다. 실행은 최대 50건으로 제한한다. `run_human50.py`는 별도로 동결한 `runs/human_keywords_v1`이 있어야 하며, 원자료 없이 실행할 수 없다. `replay_catalog.py`는 완료된 `runs/development_v2`의 최초 응답 50개를 재사용한다. + +원본 실험의 코드 스냅샷 해시는 원본에 대한 기록이다. 경로 정리 후 코드는 해시가 달라지므로 과거 실행의 protocol을 그대로 재사용하지 않는다. 새 실행을 준비하면 현재 소스와 입력으로 새 해시가 만들어진다. diff --git a/experiments/consultation-agent/evaluate.py b/experiments/consultation-agent/evaluate.py new file mode 100644 index 0000000..c8b6ff8 --- /dev/null +++ b/experiments/consultation-agent/evaluate.py @@ -0,0 +1,92 @@ +"""Prepare frozen development runs and execute real shared-contract comparisons.""" +import argparse,asyncio,hashlib,importlib.metadata,json,re,sys,time +from pathlib import Path +import contract as c +from evaluation import ARMS,summarize +from journal import Journal +from mcp_transport import MCPTools +from runner import Limits,run_case + +HERE=Path(__file__).resolve().parent;P=HERE/'inputs' +SOURCES=['contract.py','journal.py','runner.py','evaluation.py','evaluate.py','tool_backend.py','mcp_server.py','mcp_transport.py'] +def source_map():return {**{name:HERE/name for name in SOURCES},'experiment_client.py':HERE/'experiment.py'} +def sha(data):return hashlib.sha256(data).hexdigest() +def dump(path,value):path.write_text(json.dumps(value,ensure_ascii=False,indent=2)) +def folder(name): + if not re.fullmatch('[a-z0-9_-]{1,60}',name):raise ValueError('invalid_run_name') + return HERE/'runs'/name +def prepare(name,dataset): + out=folder(name);out.mkdir(parents=True,exist_ok=False) + expected=None;rows=[] + if dataset=='regression': + source=json.loads((P/'semantic_regressions.json').read_text()) + rows=[{'id':r['id'],'text':r['text']} for r in source] + expected={r['id']:{k:r[k] for k in ['required','forbidden','sentiment']} for r in source} + elif dataset=='development': + root=Path(json.loads((P/'dataset_summary.json').read_text())['source_root']) + for line in (P/'development_50.jsonl').read_text().splitlines(): + r=json.loads(line);data=(root/r['path']).read_bytes() + if sha(data)!=r['source_file_sha256']:raise ValueError('source_hash_changed') + text=json.loads(data)[r['record_index']]['consulting_content'] + text=re.sub(r'[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}','',text) + text=re.sub(r'(?',text) + rows.append({'id':r['id'],'text':text,'source_file_sha256':r['source_file_sha256']}) + else:raise ValueError('Only pre-existing development runs allowed; 2000 gated separately') + for r in rows:r['input_sha256']=sha(r['text'].encode()) + dump(out/'manifest.json',rows) + (out/'taxonomy.json').write_bytes((P/'business_keyword_alias_map.snapshot.json').read_bytes()) + if expected:dump(out/'expected_development_constraints.json',expected) + for name,path in source_map().items():(out/name).write_bytes(path.read_bytes()) + files=list(source_map())+['manifest.json','taxonomy.json']+(['expected_development_constraints.json'] if expected else []) + dump(out/'protocol.json',{'dataset':dataset,'cases':len(rows),'model':c.MODEL,'arms':ARMS,'model_call_ceiling':4,'tool_call_ceiling':6,'run_budget_usd':2.0, + 'human_gold':0,'initial_response':'shared','arm_order':'cyclic latin rotation','information':'all complete turns and full dictionary preloaded identically', + 'constraints':'same strict customer-turn-ID schema, same reconstruction/dedup/validation', + 'sdk':importlib.metadata.version('mcp'),'hashes':{name:sha((out/name).read_bytes()) for name in files}, + 'scope':'development debug only, not held-out semantic accuracy or 2000-case evaluation'}) + print(json.dumps({'prepared':str(out),'cases':len(rows)})) +async def run(name): + out=folder(name) + if (out/'completion.json').exists(): + print('Already complete; no mutation or API');return + protocol=json.loads((out/'protocol.json').read_text()) + if protocol['cases']>50:raise ValueError('Large evaluation gate not passed') + for name,h in protocol['hashes'].items(): + if sha((out/name).read_bytes())!=h:raise ValueError('snapshot_changed') + if name in source_map() and sha(source_map()[name].read_bytes())!=h:raise ValueError('source_changed_after_freeze') + if importlib.metadata.version('mcp')!=protocol['sdk']:raise ValueError('sdk_changed') + import experiment as e + api=e.client_from_env() + retrylog=(out/'retry_events.jsonl').open('a') + async def model(payload): + for attempt in range(5): + try:return await asyncio.to_thread(api,payload) + except e.APIError as ex: + retrylog.write(json.dumps({'code':str(ex),'attempt':attempt+1,'payload_hash':c.digest(payload)})+'\n');retrylog.flush() + if 'rate_limit_exceeded' not in str(ex) or attempt==4:raise + await asyncio.sleep(20) + journal=Journal(out/'state.sqlite');rows=json.loads((out/'manifest.json').read_text());tax=json.loads((out/'taxonomy.json').read_text()) + results=[];started=time.perf_counter();startup=time.perf_counter() + try: + async with MCPTools(out/'manifest.json',out/'taxonomy.json') as tools: + dump(out/'transport_startup.json',{'seconds':time.perf_counter()-startup,'sdk':protocol['sdk']}) + for index,row in enumerate(rows): + order=ARMS[index%4:]+ARMS[:index%4] + for arm in order: + result=await run_case(arm,row,tax,model,tools,journal,Limits()) + results.append(result) + dump(out/'progress.json',{'completed_arm_results':len(results),'cases_total':len(rows),'actual_receipt_estimated_usd':journal.total_cost(),'elapsed_seconds':time.perf_counter()-started}) + print(json.dumps({'id':row['id'],'arm':arm,'status':result['status'],'model_calls':result['model_calls'],'tool_calls':result['tool_calls'],'actual_usd':round(journal.total_cost(),8)}),flush=True) + (out/'results.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in results)) + receipts=journal.receipts();dump(out/'receipts.json',receipts) + # Gold-like development expectations are loaded ONLY after all model execution. + path=out/'expected_development_constraints.json';expected=json.loads(path.read_text()) if path.exists() else None + summary=summarize(results,receipts,[r['id'] for r in rows],expected) + dump(out/'summary.json',summary);dump(out/'completion.json',{'cases':len(rows),'arm_results':len(results),'actual_usd':journal.total_cost(),'elapsed_seconds':time.perf_counter()-started}) + print(json.dumps(summary,ensure_ascii=False),flush=True) + finally: + dump(out/'receipts.json',journal.receipts());journal.close();retrylog.close() +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('action',choices=['prepare','run']);parser.add_argument('--name',required=True);parser.add_argument('--dataset',choices=['regression','development']) + args=parser.parse_args() + if args.action=='prepare':prepare(args.name,args.dataset) + else:asyncio.run(run(args.name)) diff --git a/experiments/consultation-agent/evaluation.py b/experiments/consultation-agent/evaluation.py new file mode 100644 index 0000000..577ed7b --- /dev/null +++ b/experiments/consultation-agent/evaluation.py @@ -0,0 +1,39 @@ +"""Paired operational metrics; never turn structural validity into accuracy.""" +import collections +import statistics + +ARMS=['single','fixed','budget_review','adaptive'] + +def regression_checks(row,expected): + pred=row.get('prediction') or {} + names={k['name'] for k in pred.get('keywords',[])} + missing=sorted(set(expected['required'])-names) + forbidden=sorted(set(expected['forbidden'])&names) + sentiment_match=pred.get('sentiment')==expected['sentiment'] + return {'missing':missing,'forbidden':forbidden,'sentiment_match':sentiment_match,'pass':not missing and not forbidden and sentiment_match,'kind':'authored_development_constraints_not_human_gold'} + +def summarize(rows,receipts,case_ids,expected=None): + if len(case_ids)!=len(set(case_ids)):raise ValueError('duplicate_case_ids') + pairs={(r['id'],r['arm']):r for r in rows} + if len(pairs)!=len(rows) or set(pairs)!={(i,a) for i in case_ids for a in ARMS}:raise ValueError('missing_duplicate_or_extra_pairs') + for i in case_ids: + group=[pairs[(i,a)] for a in ARMS] + if len({r['base_payload_hash'] for r in group})!=1:raise ValueError('nonidentical_base_payload') + if any(r['base_raw']!=group[0]['base_raw'] for r in group):raise ValueError('nonidentical_initial_response') + output={'cases':len(case_ids),'arms':{},'human_gold_verified':0,'actual_success_receipt_cost_usd':sum(r['cost'] or 0 for r in receipts if r['status']=='COMPLETE'), + 'actual_success_calls':sum(r['status']=='COMPLETE' for r in receipts),'uncertain_calls':sum(r['status']!='COMPLETE' for r in receipts), + 'fairness':{'same_initial_payload_and_response':True,'shared_final_contract':True,'complete_input_and_dictionary_all_arms':True,'equal_max_model_calls_for_budget_review_and_adaptive':True,'same_actual_token_budget':False}, + 'limitations':['Shared initial completion design; logical arm costs include the reused base receipt','Request seconds include historical shared receipt latency; current elapsed excludes MCP session startup','No human semantic gold; authored regression checks are development-only','Equal call ceilings are not equal realized token spend'], + 'usefulness_gate':{'passed':False,'status':'pending_external_evidence_review','reason':'Structural metrics and authored development constraints alone do not establish useful adaptive-agent superiority or semantic accuracy.'}} + for arm in ARMS: + group=[pairs[(i,arm)] for i in case_ids] + output['arms'][arm]={'structural_valid':sum(r['prediction'] is not None for r in group),'validated_status':sum(r['status']=='VALIDATED' for r in group), + 'review_required':sum(r['status']=='REVIEW_REQUIRED' for r in group),'estimated_usd':sum(r['estimated_usd'] for r in group),'uncached_usd':sum(r['uncached_usd'] for r in group), + 'logical_model_calls':sum(r['model_calls'] for r in group),'tool_calls':sum(r['tool_calls'] for r in group),'tool_errors':sum(r['tool_errors'] for r in group), + 'mean_logical_request_seconds':statistics.mean(r['request_seconds'] for r in group),'mean_current_elapsed_seconds':statistics.mean(r['elapsed_seconds'] for r in group), + 'tool_seconds':sum(t.get('seconds',0) for r in group for t in r['trace'] if t['type']=='tool'), + 'stop_reasons':dict(collections.Counter(r.get('stop_reason','fixture') for r in group))} + if expected: + scores=[regression_checks(pairs[(i,arm)],expected[i]) for i in case_ids] + output['arms'][arm]['authored_development_constraints_pass']=sum(x['pass'] for x in scores) + return output diff --git a/experiments/consultation-agent/experiment.py b/experiments/consultation-agent/experiment.py new file mode 100644 index 0000000..0998993 --- /dev/null +++ b/experiments/consultation-agent/experiment.py @@ -0,0 +1,40 @@ +"""HTTP client retained from the local comparison experiment.""" +import json +import os +import ssl +import urllib.error +import urllib.request + +class APIError(Exception): + pass + +def client(key): + try: + import certifi + context = ssl.create_default_context(cafile=certifi.where()) + except ImportError: + context = ssl.create_default_context() + def call(payload): + req = urllib.request.Request('https://api.openai.com/v1/chat/completions', + data=json.dumps(payload, ensure_ascii=False).encode(), + headers={'Authorization': 'Bearer ' + key, 'Content-Type': 'application/json'}) + try: + with urllib.request.urlopen(req, timeout=60, context=context) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + try: + code = json.loads(exc.read()).get('error', {}).get('code') + except Exception: + code = 'unparsed' + # Never print response bodies, request headers or credentials. + raise APIError(f'HTTP {exc.code}; code={code}') from None + except urllib.error.URLError: + raise APIError('NETWORK_ERROR') from None + return call + + +def client_from_env(): + key = os.environ.get("OPENAI_API_KEY", "").strip() + if not key: + raise ValueError("OPENAI_API_KEY is required") + return client(key) diff --git a/experiments/consultation-agent/keyword_score.py b/experiments/consultation-agent/keyword_score.py new file mode 100644 index 0000000..c8724e3 --- /dev/null +++ b/experiments/consultation-agent/keyword_score.py @@ -0,0 +1,17 @@ +"""Keyword-only scoring against supplied labels. Sentiment intentionally excluded.""" +def score(rows,gold): + if len(rows)!=len(gold) or len({r['id'] for r in rows})!=len(rows) or {r['id'] for r in rows}!=set(gold): + raise ValueError('incomplete_or_duplicate_pairs') + tp=fp=fn=exact=review=0 + details=[] + for r in rows: + expected=set(gold[r['id']]);pred=r.get('prediction') + names={x['name'] for x in (pred or {}).get('keywords',[])} + held=r['status']!='VALIDATED' or pred is None + # All-case quality counts abstention as no delivered labels, never free success. + delivered=set() if held else names + a,b,c=len(delivered&expected),len(delivered-expected),len(expected-delivered) + tp+=a;fp+=b;fn+=c;review+=held + ok=not held and delivered==expected;exact+=ok + details.append({'id':r['id'],'exact':ok,'review_required':held,'missing':sorted(expected-delivered),'extra':sorted(delivered-expected),'raw_predicted_keywords':sorted(names)}) + return {'cases':len(rows),'tp':tp,'fp':fp,'fn':fn,'micro_precision':tp/(tp+fp) if tp+fp else None,'micro_recall':tp/(tp+fn) if tp+fn else None,'micro_f1':2*tp/(2*tp+fp+fn) if 2*tp+fp+fn else None,'exact_matches':exact,'review_required':review,'details':details} diff --git a/experiments/consultation-agent/replay_catalog.py b/experiments/consultation-agent/replay_catalog.py new file mode 100644 index 0000000..04ea0d9 --- /dev/null +++ b/experiments/consultation-agent/replay_catalog.py @@ -0,0 +1,93 @@ +"""Replay identical initial receipts; change only model-visible tool catalog.""" +import argparse,asyncio,hashlib,json,sys,time +from pathlib import Path +import contract as c +from journal import Journal +from mcp_transport import MCPTools +from runner import run_case + +H=Path(__file__).resolve().parent;P=H/'inputs';SOURCE=H/'runs/development_v2';OUT=H/'runs/catalog_v3' +def filter_catalog(tools):return [t for t in tools if t['name']!='validate_analysis'] +def import_bases(journal,receipts): + imported=[] + for r in receipts: + if r['step']!='base':continue + if r['status']!='COMPLETE' or not r['response'].get('id'):raise ValueError('unresolved_base') + journal.begin_call(r['job'],r['step'],r['payload_hash']) + journal.finish_call(r['job'],r['step'],r['payload_hash'],r['response'],r['cost'],r['seconds']) + imported.append(r['response']['id']) + return imported +class ContextTools(MCPTools): + async def __aenter__(self): + self.identity=self._backend.identity + await super().__aenter__() + self.identity+=':catalog_without_host_validator_v1' + return self + async def list_tools(self):return filter_catalog(await super().list_tools()) + +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +def dump(path,data):path.write_text(json.dumps(data,ensure_ascii=False,indent=2)) +def prepare(): + if not (SOURCE/'completion.json').exists():raise ValueError('source_incomplete') + protocol=json.loads((SOURCE/'protocol.json').read_text()) + for name,value in protocol['hashes'].items(): + if sha(SOURCE/name)!=value:raise ValueError('source_snapshot_changed') + rows=json.loads((SOURCE/'manifest.json').read_text());tax=json.loads((SOURCE/'taxonomy.json').read_text()) + receipts=json.loads((SOURCE/'receipts.json').read_text());bases={r['job']:r for r in receipts if r['step']=='base'} + if len(rows)!=50 or len(bases)!=50:raise ValueError('expected_exact_development50') + for r in rows: + payload=c.base_payload(r['text'],tax,r['id']);key=c.digest({'id':r['id'],'payload':payload}) + if bases[key]['payload_hash']!=c.digest(payload):raise ValueError('base_contract_changed') + OUT.mkdir(exist_ok=False) + for name in ['manifest.json','taxonomy.json','receipts.json','results.jsonl']:(OUT/('source_'+name)).write_bytes((SOURCE/name).read_bytes()) + sources=[H/name for name in ['replay_catalog.py','runner.py','journal.py','contract.py','mcp_transport.py','mcp_server.py','tool_backend.py']]+[H/'experiment.py'] + dump(OUT/'protocol.json',{'cases':50,'source':str(SOURCE),'intervention':'Only model catalog removes validate_analysis; full MCP server and common host validation unchanged', + 'initial_responses':'exact imported source receipts, verified payload hash','human_gold':0, + 'limits':'same runner and Limits defaults; source common prompt/model/schema unchanged', + 'development_only':True,'hashes':{str(p):sha(p) for p in sources}, + 'source_hashes':{n:sha(OUT/('source_'+n)) for n in ['manifest.json','taxonomy.json','receipts.json','results.jsonl']}}) + print('Prepared catalog-only replay of 50 existing development cases') +async def run(): + if (OUT/'completion.json').exists():print('Already complete; no mutation or API');return + protocol=json.loads((OUT/'protocol.json').read_text()) + for name,value in protocol['hashes'].items(): + if sha(Path(name))!=value:raise ValueError('frozen_source_changed') + for name,value in protocol['source_hashes'].items(): + if sha(OUT/('source_'+name))!=value:raise ValueError('frozen_receipts_changed') + rows=json.loads((OUT/'source_manifest.json').read_text());tax=json.loads((OUT/'source_taxonomy.json').read_text()) + source_receipts=json.loads((OUT/'source_receipts.json').read_text()) + before={r['id']:r for r in map(json.loads,(OUT/'source_results.jsonl').read_text().splitlines()) if r['arm']=='adaptive'} + journal=Journal(OUT/'state.sqlite') + imported=import_bases(journal,source_receipts);dump(OUT/'imported_response_ids.json',imported) + import experiment as e + api=e.client_from_env() + async def model(payload): + for attempt in range(5): + try:return await asyncio.to_thread(api,payload) + except e.APIError as ex: + with (OUT/'retry_events.jsonl').open('a') as log:log.write(json.dumps({'error':str(ex),'attempt':attempt+1})+'\n') + if 'rate_limit_exceeded' not in str(ex) or attempt==4:raise + await asyncio.sleep(20) + results=[] + try: + async with ContextTools(OUT/'source_manifest.json',OUT/'source_taxonomy.json') as tools: + for case in rows: + r=await run_case('adaptive',case,tax,model,tools,journal) + if r['base_raw']!=before[case['id']]['base_raw'] or r['base_payload_hash']!=before[case['id']]['base_payload_hash']:raise ValueError('nonidentical_initial_response') + results.append(r) + print(json.dumps({'id':r['id'],'calls':r['model_calls'],'tools':r['tool_calls'],'status':r['status']}),flush=True) + (OUT/'results.jsonl').write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in results)) + def metrics(items):return {'structural_valid':sum(r['prediction'] is not None for r in items),'review_required':sum(r['status']=='REVIEW_REQUIRED' for r in items),'model_calls':sum(r['model_calls'] for r in items),'tool_calls':sum(r['tool_calls'] for r in items),'estimated_usd':sum(r['estimated_usd'] for r in items),'uncached_usd':sum(r['uncached_usd'] for r in items)} + def signature(r): + pred=r['prediction'] or {};return (tuple(sorted(k['name'] for k in pred.get('keywords',[]))),pred.get('sentiment')) + unique=[r for r in journal.receipts() if r['status']=='COMPLETE' and r['response']['id'] not in imported] + summary={'cases':50,'before':metrics(list(before.values())),'after':metrics(results),'labels_changed':sum(signature(r)!=signature(before[r['id']]) for r in results), + 'human_gold':0,'incremental_new_success_calls':len(unique),'incremental_new_estimated_usd':sum(r['cost'] for r in unique),'imported_base_calls':len(imported), + 'same_initial_responses':True,'semantic_quality_change':'unknown','limitations':['Previously used development data','One replay per intervention; review-generation stochasticity not separated','No semantic accuracy or 2000-case gate pass']} + dump(OUT/'summary.json',summary);dump(OUT/'completion.json',{'cases':50,'status':'complete','incremental_new_estimated_usd':summary['incremental_new_estimated_usd']}) + print(json.dumps(summary),flush=True) + finally:dump(OUT/'receipts.json',journal.receipts());journal.close() +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('action',choices=['prepare','run']);args=parser.parse_args() + if args.action=='prepare':prepare() + else:asyncio.run(run()) diff --git a/experiments/consultation-agent/run_human50.py b/experiments/consultation-agent/run_human50.py new file mode 100644 index 0000000..d32990c --- /dev/null +++ b/experiments/consultation-agent/run_human50.py @@ -0,0 +1,7 @@ +"""Bounded run using the existing evaluator with the frozen reduced MCP catalog.""" +import asyncio +import evaluate +from replay_catalog import ContextTools +if __name__=='__main__': + evaluate.MCPTools=ContextTools + asyncio.run(evaluate.run('human_keywords_v1')) diff --git a/experiments/consultation-agent/tests/test_catalog.py b/experiments/consultation-agent/tests/test_catalog.py new file mode 100644 index 0000000..ea3d0e0 --- /dev/null +++ b/experiments/consultation-agent/tests/test_catalog.py @@ -0,0 +1,29 @@ +import json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +from replay_catalog import filter_catalog,import_bases +from journal import Journal + +class CatalogTests(unittest.TestCase): + def test_only_redundant_host_validator_removed(self): + tools=[{'name':n} for n in ['get_consultation_turns','validate_analysis','lookup_business_keywords']] + self.assertEqual([t['name'] for t in filter_catalog(tools)],['get_consultation_turns','lookup_business_keywords']) + self.assertEqual(len(tools),3) + def test_imported_receipts_keep_original_usage_and_identity(self): + with tempfile.TemporaryDirectory() as tmp: + j=Journal(Path(tmp)/'j.sqlite') + r={'job':'b','step':'base','payload_hash':'h','status':'COMPLETE','response':{'id':'original','usage':{'prompt_tokens':20}},'cost':.1,'seconds':2} + imported=import_bases(j,[r]) + self.assertEqual(imported,['original']) + self.assertEqual(j.begin_call('b','base','h')['response'],r['response']) + j.close() + +class CompletedRunTests(unittest.IsolatedAsyncioTestCase): + async def test_completed_run_returns_without_key_or_mutation(self): + import evaluate + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp);(p/'completion.json').write_text('{"cases":1}') + before=(p/'completion.json').read_bytes() + with patch.object(evaluate,'folder',return_value=p):await evaluate.run('completed') + self.assertEqual((p/'completion.json').read_bytes(),before) + self.assertEqual(len(list(p.iterdir())),1) diff --git a/experiments/consultation-agent/tests/test_evaluation.py b/experiments/consultation-agent/tests/test_evaluation.py new file mode 100644 index 0000000..1531609 --- /dev/null +++ b/experiments/consultation-agent/tests/test_evaluation.py @@ -0,0 +1,24 @@ +import unittest +from evaluation import summarize,regression_checks + +class EvaluationTests(unittest.TestCase): + def test_http_client_is_part_of_frozen_source_map(self): + from evaluate import source_map + self.assertEqual(source_map()['experiment_client.py'].name,'experiment.py') + def test_regression_constraint_is_not_human_accuracy(self): + row={'prediction':{'keywords':[{'name':'해지'}],'sentiment':'NEUTRAL'}} + self.assertTrue(regression_checks(row,{'required':['해지'],'forbidden':['요금제'],'sentiment':'NEUTRAL'})['pass']) + self.assertFalse(regression_checks(row,{'required':['해지'],'forbidden':[],'sentiment':'NEGATIVE'})['pass']) + def test_summary_rejects_missing_or_nonidentical_pairs(self): + row={'id':'1','arm':'single','base_payload_hash':'a'} + with self.assertRaises(ValueError):summarize([row],[],['1']) + def test_gate_does_not_pass_on_structural_perfection(self): + rows=[] + for arm in ['single','fixed','budget_review','adaptive']: + rows.append(dict(id='1',arm=arm,base_payload_hash='same',base_raw={'a':1},status='VALIDATED',prediction={'keywords':[],'sentiment':'NEUTRAL'},estimated_usd=.1,uncached_usd=.1,request_seconds=1,elapsed_seconds=1,model_calls=1,tool_calls=0,tool_errors=0,semantic_verified=False,trace=[])) + result=summarize(rows,[],['1']) + self.assertFalse(result['usefulness_gate']['passed']) + self.assertEqual(result['human_gold_verified'],0) + self.assertEqual(result['arms']['single']['structural_valid'],1) + +if __name__=='__main__':unittest.main() diff --git a/experiments/consultation-agent/tests/test_keyword_score.py b/experiments/consultation-agent/tests/test_keyword_score.py new file mode 100644 index 0000000..387c099 --- /dev/null +++ b/experiments/consultation-agent/tests/test_keyword_score.py @@ -0,0 +1,25 @@ +import unittest +import keyword_score as k + +class ScoreTests(unittest.TestCase): + def test_counts_false_positives_and_missing_output(self): + gold={'x':['A','B'],'y':['C']} + rows=[{'id':'x','prediction':{'keywords':[{'name':'A'},{'name':'D'}]},'status':'VALIDATED'}, {'id':'y','prediction':None,'status':'REVIEW_REQUIRED'}] + s=k.score(rows,gold) + self.assertEqual((s['tp'],s['fp'],s['fn']),(1,1,2)) + self.assertAlmostEqual(s['micro_f1'],0.4) + self.assertEqual(s['exact_matches'],0) + self.assertEqual(s['cases'],2) + def test_abstention_not_counted_as_exact_even_empty_gold(self): + s=k.score([{'id':'x','prediction':{'keywords':[]},'status':'REVIEW_REQUIRED'}],{'x':[]}) + self.assertEqual(s['exact_matches'],0) + self.assertEqual(s['review_required'],1) + def test_rejects_duplicate_or_missing_cases(self): + r={'id':'x','prediction':{'keywords':[]},'status':'VALIDATED'} + for rows in [[r,r],[]]: + with self.assertRaises(ValueError):k.score(rows,{'x':[]}) + def test_deduplicates_labels(self): + s=k.score([{'id':'x','prediction':{'keywords':[{'name':'A'},{'name':'A'}]},'status':'VALIDATED'}],{'x':['A']}) + self.assertEqual(s['tp'],1) + self.assertEqual(s['micro_f1'],1) + self.assertEqual(s['exact_matches'],1) diff --git a/experiments/consultation-agent/tests/test_portable_client.py b/experiments/consultation-agent/tests/test_portable_client.py new file mode 100644 index 0000000..8f0912e --- /dev/null +++ b/experiments/consultation-agent/tests/test_portable_client.py @@ -0,0 +1,17 @@ +import os +import unittest +from unittest.mock import patch +import experiment + + +class PortableClientTests(unittest.TestCase): + def test_missing_key_fails_before_http_client_creation(self): + with patch.dict(os.environ, {}, clear=True), patch.object(experiment, "client") as client: + with self.assertRaisesRegex(ValueError, "OPENAI_API_KEY"): + experiment.client_from_env() + client.assert_not_called() + + def test_environment_key_is_forwarded_without_file_access(self): + with patch.dict(os.environ, {"OPENAI_API_KEY": "fixture-key"}), patch.object(experiment, "client") as client: + self.assertIs(experiment.client_from_env(), client.return_value) + client.assert_called_once_with("fixture-key")