Repository navigation
Expand file tree
/
Copy pathasync_database.py
More file actions
360 lines (325 loc) · 13.6 KB
/
Copy pathasync_database.py
File metadata and controls
360 lines (325 loc) · 13.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
"""
异步数据库模块 - 基于 aiosqlite 的高性能实现
特性:
- 全异步操作,消除 IO 阻塞
- 批量写入优化
- 连接池管理
"""
import asyncio
import threading
import aiosqlite
from datetime import datetime
from typing import Optional, List, Dict, Any, Set, Tuple
from loguru import logger
from database import LeakedKey, KeyStatus
from key_vault import vault
from config import config as app_config
_CLAIMS: Dict[str, Tuple[threading.Lock, Set[str]]] = {}
_CLAIMS_GUARD = threading.Lock()
def _shared_claims(db_path: str) -> Tuple[threading.Lock, Set[str]]:
with _CLAIMS_GUARD:
if db_path not in _CLAIMS:
_CLAIMS[db_path] = (threading.Lock(), set())
return _CLAIMS[db_path]
class AsyncDatabase:
"""异步数据库。连接绑在当前事件循环上,验证结果一次写入。"""
def __init__(self, db_path: str = "leaked_keys.db"):
self.db_path = db_path
self._conn: Optional[aiosqlite.Connection] = None
self._conn_lock: Optional[asyncio.Lock] = None
self._write_queue: List[LeakedKey] = []
self._claim_lock, self._pending_hashes = _shared_claims(db_path)
self._queue_lock: Optional[asyncio.Lock] = None
self._batch_size = app_config.db_batch_size
self._flush_interval = app_config.db_flush_interval
self._flush_task: Optional[asyncio.Task] = None
async def init(self, *, start_flusher: bool = True):
"""在当前事件循环上打开连接并确保表结构。"""
if self._conn is not None:
return
self._conn_lock = asyncio.Lock()
self._queue_lock = asyncio.Lock()
self._conn = await aiosqlite.connect(self.db_path)
self._conn.row_factory = aiosqlite.Row
await self._conn.execute("PRAGMA journal_mode=WAL")
await self._conn.execute("PRAGMA synchronous=NORMAL")
db = self._conn
await db.execute("""
CREATE TABLE IF NOT EXISTS leaked_keys (
id INTEGER PRIMARY KEY AUTOINCREMENT,
platform TEXT NOT NULL,
api_key TEXT NOT NULL UNIQUE,
key_hash TEXT,
base_url TEXT NOT NULL,
status TEXT DEFAULT 'pending',
source_url TEXT DEFAULT '',
model_tier TEXT DEFAULT '',
rpm INTEGER DEFAULT 0,
found_time DATETIME DEFAULT CURRENT_TIMESTAMP,
verified_time DATETIME
)
""")
await db.execute("CREATE INDEX IF NOT EXISTS idx_status ON leaked_keys(status)")
await db.execute("CREATE INDEX IF NOT EXISTS idx_platform ON leaked_keys(platform)")
try:
await db.execute("ALTER TABLE leaked_keys ADD COLUMN key_hash TEXT")
except Exception:
pass
async with db.execute(
"SELECT id, api_key FROM leaked_keys WHERE key_hash IS NULL"
) as cursor:
rows = await cursor.fetchall()
for row_id, stored_key in rows:
await db.execute(
"UPDATE leaked_keys SET key_hash = ? WHERE id = ?",
(vault.hash_key(vault.decrypt(stored_key)), row_id),
)
try:
await db.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_key_hash ON leaked_keys(key_hash)"
)
except Exception as e:
logger.warning(f"key_hash 唯一索引创建失败(旧数据可能存在重复): {e}")
try:
await db.execute(
"ALTER TABLE leaked_keys ADD COLUMN disposition TEXT DEFAULT 'open'"
)
except Exception:
pass
async with db.execute("PRAGMA table_info(leaked_keys)") as cursor:
columns = {row[1] for row in await cursor.fetchall()}
for retired in (
"is_high_value", "balance", "balance_usd", "value_score",
"used_quota_usd", "total_quota_usd", "tpm", "rpd",
"has_gpt4", "has_gpt5", "has_claude_opus",
"organization", "account_name", "expiration_date", "key_type",
):
if retired in columns:
await db.execute(f"ALTER TABLE leaked_keys DROP COLUMN {retired}")
await db.execute("""
CREATE TABLE IF NOT EXISTS scanned_blobs (
file_sha TEXT PRIMARY KEY,
scan_time DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
await db.commit()
if start_flusher:
self._flush_task = asyncio.create_task(self._periodic_flush())
logger.info(f"异步数据库初始化完成: {self.db_path}")
async def connect(self) -> "AsyncDatabase":
"""在当前事件循环上打开一条新连接,与同路径实例共享 claim 集合。"""
bound = AsyncDatabase(self.db_path)
await bound.init(start_flusher=False)
return bound
async def close(self):
"""关闭数据库"""
if self._flush_task:
self._flush_task.cancel()
try:
await self._flush_task
except asyncio.CancelledError:
pass
await self._flush_queue()
if self._conn is not None:
await self._conn.close()
self._conn = None
async def _periodic_flush(self):
"""定期刷新写入队列"""
while True:
await asyncio.sleep(self._flush_interval)
await self._flush_queue()
def _row_values(self, key: LeakedKey):
key_hash = vault.hash_key(key.api_key)
return (
key.platform,
vault.encrypt(key.api_key),
key_hash,
key.base_url,
key.status,
key.source_url,
key.model_tier,
key.rpm,
key.found_time.isoformat() if key.found_time else datetime.now().isoformat(),
)
async def _flush_queue(self):
"""批量写入队列中的数据"""
if self._queue_lock is None:
return
async with self._queue_lock:
if not self._write_queue:
return
keys_to_write = self._write_queue[:]
self._write_queue.clear()
if not keys_to_write or self._conn is None or self._conn_lock is None:
return
async with self._conn_lock:
for key in keys_to_write:
try:
await self._conn.execute(
"""
INSERT OR IGNORE INTO leaked_keys
(platform, api_key, key_hash, base_url, status, source_url, model_tier, rpm, found_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
self._row_values(key),
)
with self._claim_lock:
self._pending_hashes.discard(vault.hash_key(key.api_key))
except Exception as e:
logger.debug(f"批量写入异常: {e}")
await self._conn.commit()
logger.debug(f"批量写入 {len(keys_to_write)} 条记录")
async def queue_insert(self, key: LeakedKey):
"""将 Key 加入写入队列"""
if self._queue_lock is None:
raise RuntimeError("AsyncDatabase.init() was not called")
key_hash = vault.hash_key(key.api_key)
with self._claim_lock:
self._pending_hashes.add(key_hash)
async with self._queue_lock:
self._write_queue.append(key)
if len(self._write_queue) >= self._batch_size:
asyncio.create_task(self._flush_queue())
async def flush(self):
"""立即落盘待写入队列。"""
await self._flush_queue()
async def claim_key(self, api_key: str) -> bool:
"""本轮是否该验证这条密钥。已入库或正在处理的返回 False。"""
if self._conn is None:
raise RuntimeError("AsyncDatabase.init() was not called")
key_hash = vault.hash_key(api_key)
self._claim_lock.acquire()
try:
if key_hash in self._pending_hashes:
return False
async with self._conn.execute(
"SELECT 1 FROM leaked_keys WHERE key_hash = ? LIMIT 1",
(key_hash,),
) as cursor:
if await cursor.fetchone():
return False
self._pending_hashes.add(key_hash)
return True
finally:
self._claim_lock.release()
async def release_claim(self, api_key: str) -> None:
with self._claim_lock:
self._pending_hashes.discard(vault.hash_key(api_key))
async def upsert_finding(self, key: LeakedKey) -> None:
"""验证结束后一次写入:插入或按指纹更新状态。"""
if self._conn is None or self._conn_lock is None:
raise RuntimeError("AsyncDatabase.init() was not called")
status = key.status
verified_time = datetime.now().isoformat()
values = self._row_values(key)
async with self._conn_lock:
await self._conn.execute(
"""
INSERT INTO leaked_keys
(platform, api_key, key_hash, base_url, status, source_url, model_tier, rpm, found_time, verified_time)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(key_hash) DO UPDATE SET
status=excluded.status,
model_tier=excluded.model_tier,
rpm=excluded.rpm,
verified_time=excluded.verified_time,
base_url=excluded.base_url,
source_url=excluded.source_url
""",
values + (verified_time,),
)
await self._conn.commit()
with self._claim_lock:
self._pending_hashes.discard(values[2])
async def key_exists(self, api_key: str) -> bool:
"""检查 Key 是否存在(按 key_hash 指纹查询,含未落盘的在途项)"""
key_hash = vault.hash_key(api_key)
with self._claim_lock:
if key_hash in self._pending_hashes:
return True
if self._conn is None or self._conn_lock is None:
return False
async with self._conn_lock:
async with self._conn.execute(
"SELECT 1 FROM leaked_keys WHERE key_hash = ? LIMIT 1",
(key_hash,),
) as cursor:
return await cursor.fetchone() is not None
async def is_blob_scanned(self, file_sha: str) -> bool:
"""检查文件是否已扫描"""
if self._conn is None or self._conn_lock is None:
raise RuntimeError("AsyncDatabase.init() was not called")
async with self._conn_lock:
async with self._conn.execute(
"SELECT 1 FROM scanned_blobs WHERE file_sha = ? LIMIT 1",
(file_sha,),
) as cursor:
return await cursor.fetchone() is not None
async def mark_blob_scanned(self, file_sha: str) -> bool:
"""标记文件为已扫描"""
if self._conn is None or self._conn_lock is None:
return False
try:
async with self._conn_lock:
await self._conn.execute(
"INSERT OR IGNORE INTO scanned_blobs (file_sha) VALUES (?)",
(file_sha,),
)
await self._conn.commit()
return True
except Exception:
return False
async def update_key_status(
self,
api_key: str,
status: KeyStatus,
model_tier: str = "",
rpm: int = 0,
):
"""更新状态、模型档位和 RPM。"""
if self._conn is None or self._conn_lock is None:
raise RuntimeError("AsyncDatabase.init() was not called")
async with self._conn_lock:
await self._conn.execute(
"""
UPDATE leaked_keys
SET status = ?, model_tier = ?, rpm = ?, verified_time = ?
WHERE key_hash = ?
""",
(
status.value, model_tier, rpm,
datetime.now().isoformat(), vault.hash_key(api_key),
),
)
await self._conn.commit()
async def get_stats(self) -> Dict[str, Any]:
"""获取统计信息"""
if self._conn is None or self._conn_lock is None:
raise RuntimeError("AsyncDatabase.init() was not called")
async with self._conn_lock:
async with self._conn.execute("SELECT COUNT(*) FROM leaked_keys") as cursor:
total = (await cursor.fetchone())[0]
async with self._conn.execute(
"SELECT status, COUNT(*) FROM leaked_keys GROUP BY status"
) as cursor:
statuses = {row[0]: row[1] async for row in cursor}
async with self._conn.execute(
"SELECT platform, COUNT(*) FROM leaked_keys GROUP BY platform"
) as cursor:
platforms = {row[0]: row[1] async for row in cursor}
return {
"total": total,
"valid": statuses.get('valid', 0) + statuses.get('quota_exceeded', 0),
"statuses": statuses,
"platforms": platforms,
}
def try_enable_uvloop():
"""尝试启用 uvloop 以提升性能"""
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
logger.info("uvloop 已启用")
return True
except ImportError:
logger.debug("uvloop 未安装,使用默认事件循环")
return False