feat: 策略生成成功率评测(RAG vs 无 RAG 对照)

.gitignore

@@ -11,6 +11,8 @@ build
 # Environment variables
 .env
 test-output.txt
+# 策略评测报告输出
+eval-output/
 # IDEA code analysis
 qodana.yaml
 

app/src/main/java/com/quantmore/modules/generator/dto/GenerateStrategyRequest.java

@@ -32,6 +32,9 @@ public record GenerateStrategyRequest(
     List<Long> knowledgeBaseIds,
 
     // 可选：空 = 用户默认模型
-    String providerId
+    String providerId,
+
+    // 可选：true = 跳过知识库检索（无 RAG 对照生成），null/false = 正常检索
+    Boolean skipRetrieval
 ) {
 }

app/src/main/java/com/quantmore/modules/generator/eval/EvalCase.java

@@ -0,0 +1,16 @@
+package com.quantmore.modules.generator.eval;
+
+/**
+ * 评测用例（一条策略需求描述）
+ */
+public record EvalCase(
+    String id,
+    String name,
+    String market,
+    String frequency,
+    String buyConditions,
+    String sellConditions,
+    String riskControls,
+    String difficulty
+) {
+}

app/src/main/java/com/quantmore/modules/generator/eval/EvalCaseLoader.java

@@ -0,0 +1,71 @@
+package com.quantmore.modules.generator.eval;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.RequiredArgsConstructor;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * 评测用例加载与校验：从 casesPath 读取 JSON 数组并校验字段合法性
+ */
+@Component
+@RequiredArgsConstructor
+public class EvalCaseLoader {
+
+  private static final Set<String> MARKETS = Set.of("STOCK", "ETF", "CONVERTIBLE_BOND", "FUTURES");
+  private static final Set<String> FREQUENCIES = Set.of("DAILY", "MINUTE");
+  private static final Set<String> DIFFICULTIES = Set.of("SIMPLE", "MEDIUM", "COMPLEX");
+
+  private final ResourceLoader resourceLoader;
+  private final ObjectMapper objectMapper;
+  private final EvalProperties properties;
+
+  public List<EvalCase> load() {
+    Resource resource = resourceLoader.getResource(properties.getCasesPath());
+    if (!resource.exists()) {
+      throw new IllegalStateException("评测用例文件不存在: " + properties.getCasesPath());
+    }
+    List<EvalCase> cases;
+    try (InputStream in = resource.getInputStream()) {
+      cases = objectMapper.readValue(in, new TypeReference<List<EvalCase>>() {
+      });
+    } catch (IOException e) {
+      throw new IllegalStateException("评测用例文件解析失败: " + properties.getCasesPath(), e);
+    }
+    validate(cases);
+    return cases;
+  }
+
+  private void validate(List<EvalCase> cases) {
+    if (cases == null || cases.isEmpty()) {
+      throw new IllegalStateException("评测用例为空");
+    }
+    for (EvalCase c : cases) {
+      requireNotBlank(c.id(), "id");
+      requireNotBlank(c.name(), "name");
+      requireNotBlank(c.buyConditions(), "buyConditions");
+      if (!MARKETS.contains(c.market())) {
+        throw new IllegalStateException("用例 " + c.id() + " market 非法: " + c.market());
+      }
+      if (!FREQUENCIES.contains(c.frequency())) {
+        throw new IllegalStateException("用例 " + c.id() + " frequency 非法: " + c.frequency());
+      }
+      if (!DIFFICULTIES.contains(c.difficulty())) {
+        throw new IllegalStateException("用例 " + c.id() + " difficulty 非法: " + c.difficulty());
+      }
+    }
+  }
+
+  private void requireNotBlank(String value, String field) {
+    if (value == null || value.isBlank()) {
+      throw new IllegalStateException("用例字段为空: " + field);
+    }
+  }
+}

app/src/main/java/com/quantmore/modules/generator/eval/EvalJudgeService.java

@@ -0,0 +1,94 @@
+package com.quantmore.modules.generator.eval;
+
+import com.quantmore.common.ai.LlmProviderRegistry;
+import com.quantmore.common.ai.PromptSanitizer;
+import com.quantmore.common.ai.PromptSecurityConstants;
+import com.quantmore.common.exception.BusinessException;
+import com.quantmore.common.exception.ErrorCode;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.prompt.PromptTemplate;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * LLM 评委：按 rubric 对生成代码评分，返回结构化 JSON 结果。
+ * 使用 getPlainChatClient（无工具/记忆 advisor），保证输出为可解析的纯文本 JSON。
+ */
+@Slf4j
+@Component
+public class EvalJudgeService {
+
+  private final LlmProviderRegistry registry;
+  private final PromptSanitizer sanitizer;
+  private final EvalProperties properties;
+  private final PromptTemplate template;
+
+  public EvalJudgeService(
+      LlmProviderRegistry registry,
+      PromptSanitizer sanitizer,
+      EvalProperties properties) throws IOException {
+    this.registry = registry;
+    this.sanitizer = sanitizer;
+    this.properties = properties;
+    this.template = new PromptTemplate(
+        new ClassPathResource("prompts/strategy-eval-judge.st")
+            .getContentAsString(StandardCharsets.UTF_8));
+  }
+
+  /**
+   * 评分失败抛 BusinessException(AI_SERVICE_ERROR)，由评测服务记 judgeFailed
+   */
+  public JudgeResult judge(EvalCase caseMeta, String code) {
+    String systemPrompt = template.render(Map.of(
+        "strategyName", sanitizer.sanitize(caseMeta.name()).trim(),
+        "market", caseMeta.market(),
+        "frequency", caseMeta.frequency(),
+        "buyConditions", sanitizer.sanitize(caseMeta.buyConditions()).trim(),
+        "sellConditions", sanitizeNullable(caseMeta.sellConditions()),
+        "riskControls", sanitizeNullable(caseMeta.riskControls()),
+        "generatedCode", sanitizer.wrapWithDelimiters("generated-code", code)
+    )) + PromptSecurityConstants.ANTI_INJECTION_INSTRUCTION;
+
+    String raw;
+    try {
+      raw = resolveClient().prompt()
+          .system(systemPrompt)
+          .call()
+          .chatClientResponse()
+          .chatResponse()
+          .getResult()
+          .getOutput()
+          .getText();
+    } catch (Exception e) {
+      log.error("评委评分失败: case={}, error={}", caseMeta.id(), e.getMessage(), e);
+      throw new BusinessException(ErrorCode.AI_SERVICE_ERROR, "评委评分失败: " + e.getMessage());
+    }
+    if (raw == null || raw.isBlank()) {
+      throw new BusinessException(ErrorCode.AI_SERVICE_ERROR, "评委返回内容为空");
+    }
+    return JudgeJsonParser.parse(raw);
+  }
+
+  private String sanitizeNullable(String value) {
+    return value == null ? "" : sanitizer.sanitize(value).trim();
+  }
+
+  private ChatClient resolveClient() {
+    String providerId = properties.getJudgeProvider();
+    return (providerId == null || providerId.isBlank())
+        ? registry.getPlainChatClient()
+        : registry.getPlainChatClient(providerId);
+  }
+
+  public record JudgeResult(double score, boolean passed, List<JudgeIssue> issues) {
+  }
+
+  public record JudgeIssue(String dimension, String comment) {
+  }
+}

app/src/main/java/com/quantmore/modules/generator/eval/EvalProperties.java

@@ -0,0 +1,46 @@
+package com.quantmore.modules.generator.eval;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.time.Duration;
+
+/**
+ * 策略生成评测配置（均可用 APP_EVAL_* 环境变量覆盖）
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix = "app.eval")
+public class EvalProperties {
+
+  /** 是否启用评测：APP_EVAL_ENABLED=true 时 bootRun 跑完评测写报告后自动退出 */
+  private boolean enabled = false;
+
+  /** 用例文件路径，支持 classpath: / file: 前缀 */
+  private String casesPath = "classpath:eval/strategy-eval-cases.json";
+
+  /** 生成用 provider id（空 = 用户默认/全局默认） */
+  private String generateProvider;
+
+  /** 评委用 provider id（空 = 全局默认） */
+  private String judgeProvider;
+
+  /** 报告输出目录（相对 bootRun 工作目录） */
+  private String outputDir = "eval-output";
+
+  /** 评委通过分数线 */
+  private double judgePassScore = 70.0;
+
+  /** python 解释器路径 */
+  private String pythonBin = "python3";
+
+  /** 单次 python 语法检查超时 */
+  private Duration pythonTimeout = Duration.ofSeconds(10);
+
+  /** 等待知识库向量化就绪的超时 */
+  private Duration vectorWaitTimeout = Duration.ofSeconds(120);
+
+  /** 评测结束后是否清理本次评测写入的生成记录 */
+  private boolean cleanupRecords = true;
+}

app/src/main/java/com/quantmore/modules/generator/eval/EvalReport.java

@@ -0,0 +1,139 @@
+package com.quantmore.modules.generator.eval;
+
+import java.util.List;
+
+/**
+ * 评测报告数据结构（嵌套 record，聚合逻辑见 EvalSummary.of）
+ */
+public final class EvalReport {
+
+  private EvalReport() {
+  }
+
+  /**
+   * 单个分支（RAG 或无 RAG）的执行结果
+   */
+  public record BranchResult(
+      boolean ragEnabled,
+      boolean generationOk,
+      String generationError,
+      Long generationId,
+      long generationMs,
+      PythonSyntaxCheckService.SyntaxCheckResult syntax,
+      boolean judgeOk,
+      EvalJudgeService.JudgeResult judge,
+      String judgeRaw
+  ) {
+  }
+
+  /**
+   * 单个用例的 RAG / no-RAG 对照结果
+   */
+  public record CaseResult(EvalCase caseMeta, BranchResult rag, BranchResult noRag) {
+  }
+
+  /**
+   * 汇总统计（纯函数聚合，便于单测）
+   */
+  public record EvalSummary(
+      int totalCases,
+      int ragPassed,
+      int noRagPassed,
+      double ragAvgScore,
+      double noRagAvgScore,
+      int ragSyntaxPassed,
+      int noRagSyntaxPassed,
+      int generationFailures,
+      int judgeFailures,
+      int py35WarningCount,
+      double scoreDelta
+  ) {
+
+    public static EvalSummary of(List<CaseResult> results, double passScore) {
+      int ragPassed = 0;
+      int noRagPassed = 0;
+      int ragSyntaxPassed = 0;
+      int noRagSyntaxPassed = 0;
+      int generationFailures = 0;
+      int judgeFailures = 0;
+      int py35WarningCount = 0;
+      double ragScoreSum = 0;
+      int ragScoreCount = 0;
+      double noRagScoreSum = 0;
+      int noRagScoreCount = 0;
+
+      for (CaseResult result : results) {
+        ragPassed += passed(result.rag(), passScore) ? 1 : 0;
+        noRagPassed += passed(result.noRag(), passScore) ? 1 : 0;
+        ragSyntaxPassed += syntaxPassed(result.rag()) ? 1 : 0;
+        noRagSyntaxPassed += syntaxPassed(result.noRag()) ? 1 : 0;
+        generationFailures += result.rag().generationOk() ? 0 : 1;
+        generationFailures += result.noRag().generationOk() ? 0 : 1;
+        judgeFailures += judgeFailed(result.rag()) ? 1 : 0;
+        judgeFailures += judgeFailed(result.noRag()) ? 1 : 0;
+        py35WarningCount += warnings(result.rag()).size();
+        py35WarningCount += warnings(result.noRag()).size();
+        if (result.rag().judgeOk()) {
+          ragScoreSum += result.rag().judge().score();
+          ragScoreCount++;
+        }
+        if (result.noRag().judgeOk()) {
+          noRagScoreSum += result.noRag().judge().score();
+          noRagScoreCount++;
+        }
+      }
+
+      double ragAvgScore = ragScoreCount == 0 ? 0 : ragScoreSum / ragScoreCount;
+      double noRagAvgScore = noRagScoreCount == 0 ? 0 : noRagScoreSum / noRagScoreCount;
+      return new EvalSummary(
+          results.size(),
+          ragPassed,
+          noRagPassed,
+          ragAvgScore,
+          noRagAvgScore,
+          ragSyntaxPassed,
+          noRagSyntaxPassed,
+          generationFailures,
+          judgeFailures,
+          py35WarningCount,
+          ragAvgScore - noRagAvgScore
+      );
+    }
+
+    private static boolean passed(BranchResult branch, double passScore) {
+      return branch.generationOk()
+          && syntaxPassed(branch)
+          && branch.judgeOk()
+          && branch.judge().score() >= passScore;
+    }
+
+    private static boolean syntaxPassed(BranchResult branch) {
+      return branch.generationOk() && "PASS".equals(branch.syntax().status());
+    }
+
+    private static boolean judgeFailed(BranchResult branch) {
+      return branch.generationOk() && !branch.judgeOk();
+    }
+
+    private static List<String> warnings(BranchResult branch) {
+      return branch.syntax() == null ? List.of() : branch.syntax().py35Warnings();
+    }
+  }
+
+  /**
+   * 完整评测报告
+   */
+  public record FullReport(
+      String runAt,
+      String evalUser,
+      String generateProvider,
+      String judgeProvider,
+      String pythonVersion,
+      boolean kbReady,
+      int kbCompleted,
+      int kbFailed,
+      List<CaseResult> results,
+      EvalSummary summary
+  ) {
+  }
+}