Repository navigation
Expand file tree
/
Copy pathvalidator_optimized.py
More file actions
617 lines (514 loc) · 22.3 KB
/
Copy pathvalidator_optimized.py
File metadata and controls
617 lines (514 loc) · 22.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
"""
验证器模块 - 优化版 v2.2
v2.2 新增优化:
1. 智能缓存系统 - 3层缓存架构,减少重复验证
2. 域名健康度追踪 - 避免验证死域名
v2.1 优化(保留):
1. 连接池管理 - 复用 HTTP 连接
2. 智能重试机制 - 指数退避
3. 改进的错误分类
4. 性能监控增强
"""
import asyncio
import inspect
import queue as thread_queue
import ssl
from functools import partial
from typing import Tuple, Optional, Dict, Any
from datetime import datetime
from dataclasses import dataclass
import aiohttp
from aiohttp import ClientTimeout
from loguru import logger
from config import (
config,
PROTECTED_DOMAINS,
SAFE_HTTP_STATUS_CODES,
CIRCUIT_BREAKER_HTTP_CODES,
CIRCUIT_BREAKER_FAILURE_THRESHOLD,
CIRCUIT_BREAKER_RECOVERY_TIMEOUT,
CIRCUIT_BREAKER_HALF_OPEN_REQUESTS
)
from database import Database, LeakedKey, KeyStatus
from utils import relay_url_is_allowed
from connection_pool import get_connection_pool
from retry_handler import RetryHandler, RetryConfig, ErrorType
from cache_manager import CacheManager, CacheConfig, get_cache_manager
# 导入原有的熔断器和工具函数
from validator import (
CircuitBreaker,
circuit_breaker,
mask_key,
ValidationResult,
MAX_CONCURRENCY,
REQUEST_TIMEOUT,
RPM_ENTERPRISE_THRESHOLD,
RPM_FREE_TRIAL_THRESHOLD
)
class OptimizedAsyncValidator:
"""
优化版异步验证器
v2.2 新特性:
- 智能缓存系统(3层缓存架构)
- 域名健康度追踪
v2.1 特性(保留):
- 使用连接池复用 HTTP 连接
- 智能重试机制(指数退避)
- 改进的性能监控
"""
def __init__(self, db: Database, dashboard=None,
cache_config: Optional[CacheConfig] = None):
self.db = db
self.dashboard = dashboard
self.semaphore = asyncio.Semaphore(MAX_CONCURRENCY)
self._circuit_breaker = circuit_breaker
# v2.1: 连接池和重试处理器
self._connection_pool = None
self._retry_handler = RetryHandler(RetryConfig(
max_retries=3,
initial_delay=1.0,
max_delay=10.0,
exponential_base=2.0,
jitter=True
))
self._cache_manager: Optional[CacheManager] = None
self._cache_config = cache_config
# 性能统计
self._stats = {
'total_validations': 0,
'successful_validations': 0,
'failed_validations': 0,
'retried_validations': 0,
'connection_reused': 0,
'cache_hits': 0,
'cache_misses': 0,
'dead_domain_skipped': 0,
}
async def init_cache(self):
"""
初始化缓存管理器(v2.2 新增)
必须在使用验证器前调用
"""
if self._cache_manager is None:
self._cache_manager = await get_cache_manager(self._cache_config)
logger.info("缓存管理器已初始化")
async def _get_session(self, url: str) -> aiohttp.ClientSession:
"""
获取 session(使用连接池)
优化:为不同域名维护独立的 session,提高连接复用率
"""
if self._connection_pool is None:
self._connection_pool = await get_connection_pool()
session = await self._connection_pool.get_session(url)
self._stats['connection_reused'] += 1
return session
async def close(self):
"""关闭资源"""
# 连接池由全局管理,不需要在这里关闭
# 缓存管理器也由全局管理
pass
def _get_proxy(self) -> Optional[str]:
"""获取代理 URL"""
return config.proxy_url if config.proxy_url else None
def _log(self, message: str, level: str = "INFO"):
"""输出日志"""
if self.dashboard:
self.dashboard.add_log(message, level)
def _try_url_variants(self, base_url: str, path: str) -> list:
"""生成 URL 变体"""
base_url = base_url.rstrip('/')
path = path.lstrip('/')
variants = [f"{base_url}/{path}"]
if '/v1' not in base_url:
variants.append(f"{base_url}/v1/{path}")
if '/v1' in base_url:
base_without_v1 = base_url.replace('/v1', '')
variants.append(f"{base_without_v1}/v1/{path}")
return variants
# ========================================================================
# 熔断器集成方法
# ========================================================================
async def _check_circuit_breaker(self, base_url: str) -> Optional[ValidationResult]:
"""检查熔断器状态"""
if not config.circuit_breaker_enabled:
return None
if not await self._circuit_breaker.is_allowed(base_url):
self._log(f"熔断中: {base_url[:30]}...", "WARN")
return ValidationResult(KeyStatus.CONNECTION_ERROR, "域名熔断中")
return None
async def _record_circuit_result(
self,
url: str,
success: bool = False,
error: Exception = None,
http_status: int = None
):
"""记录请求结果到熔断器"""
if not config.circuit_breaker_enabled:
return
if success:
await self._circuit_breaker.record_success(url)
else:
await self._circuit_breaker.record_failure(url, error, http_status)
def _is_likely_valid_relay(self, base_url: str) -> bool:
return relay_url_is_allowed(base_url)
# ========================================================================
# 优化的验证方法
# ========================================================================
async def _make_request_with_retry(
self,
method: str,
url: str,
headers: dict,
proxy: Optional[str] = None,
json_data: Optional[dict] = None
) -> aiohttp.ClientResponse:
"""
发送 HTTP 请求(带重试)
新增:使用重试处理器自动重试临时错误
"""
session = await self._get_session(url)
async def _do_request():
if method.upper() == 'GET':
return await session.get(url, headers=headers, proxy=proxy)
elif method.upper() == 'POST':
return await session.post(url, headers=headers, json=json_data, proxy=proxy)
else:
raise ValueError(f"不支持的 HTTP 方法: {method}")
# 使用重试处理器
try:
response = await self._retry_handler.execute_with_retry(_do_request)
return response
except Exception as e:
# 记录重试统计
error_type = self._retry_handler.classify_error(e)
if error_type == ErrorType.RETRYABLE:
self._stats['retried_validations'] += 1
raise
async def validate_openai(self, api_key: str, base_url: str) -> ValidationResult:
"""
异步验证 OpenAI / 中转站
v2.2 优化:
1. 智能缓存 - 检查缓存避免重复验证
2. 域名健康度 - 跳过死域名
v2.1 优化(保留):
1. 使用连接池复用连接
2. 智能重试临时错误
3. 改进的错误处理
"""
self._stats['total_validations'] += 1
# 预检查 base_url 有效性
if not self._is_likely_valid_relay(base_url):
self._stats['failed_validations'] += 1
return ValidationResult(KeyStatus.INVALID, "base_url 无效")
if not base_url:
base_url = config.default_base_urls["openai"]
# v2.2: 检查缓存
if self._cache_manager:
cached_result = await self._cache_manager.get_validation_result(api_key, base_url)
if cached_result:
self._stats['cache_hits'] += 1
self._log(f"缓存命中: {mask_key(api_key)}", "DEBUG")
return ValidationResult(
KeyStatus(cached_result['status']),
cached_result.get('info', ''),
cached_result.get('model_tier') or "",
cached_result.get('rpm', 0),
)
self._stats['cache_misses'] += 1
# v2.2: 检查域名健康度
if self._cache_manager:
is_dead = await self._cache_manager.is_domain_dead(base_url)
if is_dead:
self._stats['dead_domain_skipped'] += 1
self._log(f"跳过死域名: {base_url[:30]}...", "WARN")
return ValidationResult(KeyStatus.CONNECTION_ERROR, "域名已死")
# 熔断器检查
circuit_result = await self._check_circuit_breaker(base_url)
if circuit_result:
self._stats['failed_validations'] += 1
# 记录域名失败
if self._cache_manager:
await self._cache_manager.record_domain_failure(base_url)
return circuit_result
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
proxy = self._get_proxy()
model_tier = "GPT-3.5"
rpm = 0
models_list = []
# Step 1: GET /models(带重试)
for url in self._try_url_variants(base_url, "models"):
try:
async with await self._make_request_with_retry('GET', url, headers, proxy) as resp:
# 提取 RPM
rpm = int(resp.headers.get('x-ratelimit-limit-requests', 0))
if resp.status == 200:
# 记录成功
await self._record_circuit_result(url, success=True)
self._stats['successful_validations'] += 1
# v2.2: 记录域名成功
if self._cache_manager:
await self._cache_manager.record_domain_success(base_url)
data = await resp.json()
models_list = [m.get("id", "") for m in data.get("data", [])]
# 记录模型档位,不据此标记高价值
for m in models_list:
if any(hv in m.lower() for hv in ['gpt-4', 'gpt-4o']):
model_tier = "GPT-4"
break
model_names = [m[:15] for m in models_list[:3]]
info = f"{len(models_list)}模型: {', '.join(model_names)}"
# RPM 透视标记
rpm_tier = ""
if rpm >= RPM_ENTERPRISE_THRESHOLD:
rpm_tier = "Enterprise"
elif rpm > 0 and rpm <= RPM_FREE_TRIAL_THRESHOLD:
rpm_tier = "Free Trial"
if rpm_tier:
info = f"{info} [{rpm_tier}]"
result = ValidationResult(KeyStatus.VALID, info, model_tier, rpm)
# 缓存只留状态、说明、模型档位和 RPM。不记高价值。
if self._cache_manager:
await self._cache_manager.set_validation_result(
api_key, base_url,
{
'status': result.status.value,
'info': result.info,
'model_tier': result.model_tier,
'rpm': result.rpm,
}
)
return result
elif resp.status == 429:
await self._record_circuit_result(url, http_status=429)
self._stats['failed_validations'] += 1
# v2.2: 记录域名失败
if self._cache_manager:
await self._cache_manager.record_domain_failure(base_url)
return ValidationResult(KeyStatus.QUOTA_EXCEEDED, "配额耗尽")
elif resp.status in CIRCUIT_BREAKER_HTTP_CODES:
await self._record_circuit_result(url, http_status=resp.status)
self._stats['failed_validations'] += 1
# v2.2: 记录域名失败
if self._cache_manager:
await self._cache_manager.record_domain_failure(base_url)
return ValidationResult(KeyStatus.CONNECTION_ERROR, f"网关错误 {resp.status}")
elif resp.status == 401:
self._stats['failed_validations'] += 1
return ValidationResult(KeyStatus.INVALID, "认证失败")
else:
continue
except asyncio.TimeoutError:
await self._record_circuit_result(url, error=asyncio.TimeoutError())
continue
except aiohttp.ClientError as e:
await self._record_circuit_result(url, error=e)
continue
except Exception as e:
logger.debug(f"验证异常: {e}")
continue
# 所有 URL 变体都失败
self._stats['failed_validations'] += 1
# v2.2: 记录域名失败
if self._cache_manager:
await self._cache_manager.record_domain_failure(base_url)
return ValidationResult(KeyStatus.CONNECTION_ERROR, "连接失败")
def get_stats(self) -> dict:
"""获取性能统计"""
stats = self._stats.copy()
# v2.1: 添加重试处理器统计
retry_stats = self._retry_handler.get_stats()
stats.update({
'retry_success_rate': retry_stats.get('success_rate', 0),
'retry_attempts': retry_stats.get('total_attempts', 0),
})
# v2.1: 添加连接池统计
if self._connection_pool:
pool_stats = self._connection_pool.get_stats()
stats.update({
'active_sessions': pool_stats.get('active_sessions', 0),
'domains': pool_stats.get('domains', []),
})
# v2.2: 添加缓存统计
if self._cache_manager:
cache_stats = self._cache_manager.get_stats()
stats['cache'] = cache_stats
# 计算缓存命中率
total_cache_requests = self._stats['cache_hits'] + self._stats['cache_misses']
if total_cache_requests > 0:
stats['cache_hit_rate'] = (
self._stats['cache_hits'] / total_cache_requests * 100
)
else:
stats['cache_hit_rate'] = 0.0
return stats
# ============================================================================
# v2.2: 验证器启动入口(供 main_v2.2 调用)
# ============================================================================
def _safe_dashboard(dashboard, method: str, *args, **kwargs):
"""容错调用 Dashboard 方法(不同版本 Dashboard 的 API 有差异)"""
if dashboard is None:
return
try:
getattr(dashboard, method)(*args, **kwargs)
except Exception:
pass
async def take_queued_result(result_queue, timeout: float = 1.0):
"""从扫描线程的同步队列或 asyncio.Queue 取出一条结果。"""
getter = result_queue.get
if inspect.iscoroutinefunction(getter):
item = await asyncio.wait_for(getter(), timeout=timeout)
if item is None:
raise asyncio.TimeoutError
return item
if isinstance(result_queue, asyncio.Queue):
return await asyncio.wait_for(result_queue.get(), timeout=timeout)
loop = asyncio.get_running_loop()
try:
return await loop.run_in_executor(
None, partial(result_queue.get, timeout=timeout)
)
except thread_queue.Empty:
raise asyncio.TimeoutError
async def optimized_validator_worker(
result_queue,
async_db,
stop_event,
dashboard=None,
worker_id: int = 0,
cache_config: Optional[CacheConfig] = None,
):
"""
v2.2 优化版验证工作协程
- OpenAI / 中转站 Key 走缓存 + 域名健康度 + 智能重试路径
- 其余平台走 validate_key_async 通用分发
- 入库使用完整 LeakedKey(ScanResult 入队后补 status / model_tier / rpm)
"""
from validator import validate_key_async, CircuitBreaker
db = await async_db.connect()
validator = OptimizedAsyncValidator(db, dashboard, cache_config)
await validator.init_cache()
circuit_breaker = CircuitBreaker()
processed_count = 0
logger.info(f"[Validator-{worker_id}] v2.2 优化版验证器启动")
try:
while not stop_event.is_set():
try:
result = await take_queued_result(result_queue, timeout=1.0)
except asyncio.TimeoutError:
continue
except Exception as e:
logger.error(f"[Validator-{worker_id}] 队列读取错误: {e}")
continue
try:
masked = mask_key(result.api_key)
platform = (result.platform or "openai").lower()
if not await db.claim_key(result.api_key):
logger.debug(f"[Validator-{worker_id}] Key 已存在,跳过: {masked}")
_safe_dashboard(dashboard, "increment_stat", "skipped_duplicate")
continue
if platform in ("openai", "relay"):
vr = await validator.validate_openai(result.api_key, result.base_url or "")
status = vr.status
model_tier = vr.model_tier
rpm = vr.rpm
else:
status, model_tier, rpm = await validate_key_async(
result.platform,
result.api_key,
result.base_url,
circuit_breaker=circuit_breaker
)
status_value = status.value if isinstance(status, KeyStatus) else status
await db.upsert_finding(LeakedKey(
platform=result.platform,
api_key=result.api_key,
base_url=result.base_url or "",
source_url=getattr(result, "source_url", "") or "",
status=status_value,
model_tier=model_tier or "",
rpm=rpm or 0,
))
processed_count += 1
source_url = getattr(result, "source_url", "") or ""
if status in (KeyStatus.VALID, KeyStatus.QUOTA_EXCEEDED):
try:
from notify_integration import disclose_finding
await disclose_finding(
db.db_path, result.platform, result.api_key, source_url,
)
except Exception as notify_error:
logger.error(f"[Validator-{worker_id}] 处置通知失败: {notify_error}")
source_short = source_url.rsplit('/', 1)[-1]
if status == KeyStatus.VALID:
_safe_dashboard(dashboard, "increment_stat", "valid_keys")
_safe_dashboard(dashboard, "add_valid_key",
platform=result.platform,
masked_key=masked,
source=source_short)
_safe_dashboard(dashboard, "add_log",
f"✓ 有效! {model_tier or platform.upper()} {masked}",
"VALID")
elif status == KeyStatus.QUOTA_EXCEEDED:
_safe_dashboard(dashboard, "increment_stat", "quota_exceeded")
_safe_dashboard(dashboard, "add_log", f"⚠ 配额耗尽 {masked}", "WARN")
elif status == KeyStatus.CONNECTION_ERROR:
_safe_dashboard(dashboard, "increment_stat", "connection_errors")
_safe_dashboard(dashboard, "add_log", f"✗ 连接失败 {masked}", "ERROR")
else:
_safe_dashboard(dashboard, "increment_stat", "invalid_keys")
if processed_count % 10 == 0:
logger.info(f"[Validator-{worker_id}] 已处理 {processed_count} 个 Key")
except Exception as e:
try:
await db.release_claim(result.api_key)
except Exception:
pass
logger.error(f"[Validator-{worker_id}] 验证异常: {e}")
_safe_dashboard(dashboard, "add_log", f"[✗] 验证错误: {str(e)[:50]}", "ERROR")
except asyncio.CancelledError:
logger.info(f"[Validator-{worker_id}] 收到取消信号")
finally:
await db.close()
logger.info(f"[Validator-{worker_id}] 验证器停止,共处理 {processed_count} 个 Key")
def start_validators(
result_queue,
async_db,
stop_event,
dashboard=None,
num_workers: int = 2,
cache_config: Optional[CacheConfig] = None,
):
"""
启动 v2.2 优化版验证器(每个线程内跑独立事件循环)
Args:
result_queue: asyncio.Queue 或 queue.Queue 扫描结果队列
async_db: AsyncDatabase,每个 worker 在本线程事件循环上 connect()
stop_event: threading.Event 停止信号
dashboard: Dashboard UI 实例
num_workers: 验证器线程数
cache_config: v2.2 缓存配置
Returns:
List[threading.Thread]
"""
import threading
threads = []
def _run(worker_id: int):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(
optimized_validator_worker(
result_queue, async_db, stop_event,
dashboard=dashboard, worker_id=worker_id,
cache_config=cache_config
)
)
finally:
loop.close()
for i in range(num_workers):
t = threading.Thread(target=_run, args=(i,), daemon=True, name=f"Validator-{i}")
t.start()
threads.append(t)
logger.info(f"已启动 {num_workers} 个 v2.2 优化版验证器")
return threads