diff --git a/.gitignore b/.gitignore index f3bd82a7d2..5890de5dcb 100644 --- a/.gitignore +++ b/.gitignore @@ -307,3 +307,6 @@ docs-ui-refactor/ /.ds-sync/ /.design-sync/ /src/frontend/client/.ds-sync-gen/ + +# Packaged Linsight skill bundles (scripts/pack_linsight_skill.sh output) +/dist/ diff --git "a/docs/PRD/2.6 \347\201\265\346\200\235 deepagents \350\277\201\347\247\273 PRD/Anthropic office skills \344\276\235\350\265\226\345\256\241\350\256\241\344\270\216 bisheng \347\216\257\345\242\203\347\274\272\345\217\243.md" "b/docs/PRD/2.6 \347\201\265\346\200\235 deepagents \350\277\201\347\247\273 PRD/Anthropic office skills \344\276\235\350\265\226\345\256\241\350\256\241\344\270\216 bisheng \347\216\257\345\242\203\347\274\272\345\217\243.md" new file mode 100644 index 0000000000..03911d31c1 --- /dev/null +++ "b/docs/PRD/2.6 \347\201\265\346\200\235 deepagents \350\277\201\347\247\273 PRD/Anthropic office skills \344\276\235\350\265\226\345\256\241\350\256\241\344\270\216 bisheng \347\216\257\345\242\203\347\274\272\345\217\243.md" @@ -0,0 +1,228 @@ +# Anthropic 官方 office skills（xlsx / pptx / docx）依赖审计与 bisheng 环境缺口 + +> **调研日期**：2026-08-11 +> **对象**：`github.com/anthropics/skills` 的 `skills/xlsx`、`skills/pptx`、`skills/docx`（浅克隆全量源码逐文件读取，非凭记忆） +> **对照基线**：`src/backend/pyproject.toml` + `uv.lock`、`src/backend/base.Dockerfile`、`local_executor.py` / `e2b_executor.py`、114 手动部署环境、E2B `code-interpreter-v1` 默认模板 +> **方法**：多 agent 并行提取 + 对抗式核验（关键结论均要求一手来源；被证伪的原始判断已按修正表述改写） +> **姊妹篇**：[灵思代码执行沙箱选型调研（私有化场景）](./灵思代码执行沙箱选型调研（私有化场景）.md) + +**一句话结论**：依赖缺口按 skill 分档差别极大 —— **xlsx 在官方镜像上主干本来就能跑通，docx 只死「创建」一半，只有 pptx 是真·大面积失效**；但即使依赖全部补齐，官方 skill 里 11 条「只对 Claude 成立」的假设也不会消失，因此**唯一现实选项是做适配版**（`linsight-skills/bisheng-pptx/` 已验证该路线），而非原样导入 —— 后者还额外撞上 LICENSE 的专有条款红线。 + +--- + +## 1. 三个 skill 的依赖全景 + +标记含义：**阻塞** = 缺了主干路径直接死；**降级** = 有替代路径，功能打折；**可选** = 有 which/try 守卫或走不到；**—** = 该 skill 完全不引用。 + +| 依赖项 | 类型 | xlsx | pptx | docx | 关键度（取三者最高） | 用途 | +|---|---|---|---|---|---|---| +| openpyxl | py | **阻塞** | — | — | 阻塞 | xlsx 创建/编辑主干 + `recalc.py:21` | +| pandas | py | 降级 | — | — | 降级 | xlsx 批量进出，openpyxl 可完全替代 | +| defusedxml | py | 可选 | **阻塞** | **阻塞** | 阻塞 | `clean.py` / `thumbnail.py` / `merge_runs.py` / `comment.py` / `office/validate.py` 首行 import | +| lxml | py | 可选 | **阻塞** | **阻塞** | 阻塞 | `validators/base.py:11` XSD 引擎唯一实现 | +| markitdown（CLI 形态） | py/bin | 降级 | **阻塞** | — | 阻塞 | pptx 唯一文本读取路径 + Content QA；xlsx 仅 quick look | +| Pillow | py | — | 降级 | — | 降级 | 仅 `thumbnail.py` 拼标注网格 | +| python-pptx | py | — | 可选 | — | 可选 | pptx 里被官方**主动劝退**的次选路径 | +| **pptxgenjs** | node | — | **阻塞** | — | 阻塞 | pptx「创建新 deck」官方主推，SKILL.md 近半篇幅为其 footgun | +| **docx (docx-js)** | node | — | — | **阻塞** | 阻塞 | docx「创建新文档」官方主推 | +| sharp / react / react-dom / react-icons | node | — | 降级 | — | 降级 | pptx 图标流水线（sharp 带 libvips 原生二进制，最难离线） | +| node | bin | — | **阻塞** | **阻塞** | 阻塞 | 跑上面两条 JS 主干 | +| npm | bin | — | 降级 | 降级 | 降级 | SKILL.md 唯一自愈路径，隐含 registry 可达 | +| **soffice + Calc** | bin | **阻塞** | — | — | 阻塞 | `recalc.py` 公式重算（SKILL.md 标 mandatory） | +| **soffice + Impress** | bin | — | **阻塞** | — | 阻塞 | pptx→pdf（视觉 QA 与 `thumbnail.py`） | +| **soffice + Writer** | bin | — | — | **阻塞** | 阻塞 | docx→pdf 渲染自检 + `accept_changes.py` 宏 | +| pdftoppm (poppler-utils) | bin | — | **阻塞** | **阻塞** | 阻塞 | PDF→JPG，两个 skill 的视觉 QA 唯一实现 | +| pandoc | bin | — | — | **阻塞** | 阻塞 | docx 唯一读取路径 `pandoc -t markdown` | +| zip | bin | — | **阻塞** | **阻塞** | 阻塞 | 「解包→改 XML→回包」主干的回包步 | +| unzip | bin | — | 可选 | **阻塞** | 阻塞 | docx SKILL.md:52 用 `unzip`；pptx 用 python zipfile 解包 | +| gcc | bin | 可选 | 可选 | 可选 | 可选 | 仅 AF_UNIX 被禁时现场编 LD_PRELOAD shim，容器内永不触发 | +| git | bin | 可选 | 可选 | 可选 | 可选 | `redlining.py` 逐字 diff，有 try/except 兜底；仅 docx 分支可达 | +| timeout / gtimeout / grep / find | bin | 可选 | 可选 | 可选 | 可选 | 均有 which 守卫或基础镜像自带 | +| ghostscript / ImageMagick / pdftk | bin | — | — | — | **零引用** | 三个 skill 全文 grep 无命中，**不要装** | + +共享 `scripts/office/` 树（三份逐字节相同，md5 已核）： + +| 共享文件 | md5 | xlsx | pptx | docx | +|---|---|---|---|---| +| `office/soffice.py` | 2508a6df… | 用（recalc） | 用（转 pdf） | 用（转 pdf） | +| `office/validate.py` | 5e0197ec… | **空转**（`case "xlsx"` 直接 `sys.exit(0)`） | 用（required） | 用 | +| `office/validators/`（base/docx/pptx/redlining） | base=566ce69b… | 不可达 | 用 | 用 | +| `office/helpers/__init__.py` | e3ed7338… | 不可达 | 用（safe_extract/rezip） | 用 | +| `office/schemas/`（43 个 XSD，976,913 B） | — | **完全不可达** | 用 pml/dml-main/dml-chart/opc-* | 用 wml/opc-*/microsoft/* | + +各 skill 独占资产：xlsx = `recalc.py`（308 行）；pptx = `clean.py` / `thumbnail.py` / `add_slide.py` + `helpers/pptx_*.py`；docx = `merge_runs.py` / `comment.py` / `accept_changes.py` + `scripts/templates/`（5 个批注骨架 XML，其中 `people.xml` 是死资产，`comment.py` 从不 copy 它）。 + +--- + +## 2. 缺口矩阵（核心交付物） + +> ⚠️ **必读的环境区分**：114 是 RHEL 上手动装的开发测试机，与 `src/backend/base.Dockerfile` 构建出的官方发布镜像**不是同一套依赖基线**，两者的缺口面**互补而非包含**——114 有 node、官方镜像没有；官方镜像有完整 libreoffice 元包（含 Impress/Calc）和 pandoc 3.6.4，114 只有 libreoffice-writer。**"在 114 上装好跑通了"绝不等于"发布版能跑"**，反之亦然。任何验收都必须在两套环境各跑一遍，或者只以镜像为准。 + +| 依赖项 | bisheng 官方镜像 | 114 手动部署 | E2B `code-interpreter-v1` | 缺口结论 | +|---|---|---|---|---| +| openpyxl / pandas / Pillow / lxml / python-pptx / python-docx | ✅ pyproject 直接依赖（lxml 为 transitive） | ✅ 同 venv | ⚠️ 待验证，非模板承诺内容 | 镜像/114 无缺口；E2B 不可依赖 | +| defusedxml | ❌ pyproject + uv.lock 零命中 | ❌ | ⚠️ 待验证 | **全线缺失**，纯 python 包，最便宜的一修 | +| markitdown | ❌ 零命中 | ❌ 命令不存在 | ⚠️ 待验证 | 全线缺失；pptx 读取路径断 | +| soffice 二进制 | ✅ apt `libreoffice` 元包 | ✅ | ❌ 模板不承诺 | — | +| └ Calc（xlsx） | ✅ 元包 Depends 带入 | ❌ **只装了 writer** | ❌ | 114 上 `recalc.py` 必失败，且**伪装成 `soffice exited N` 或"never rewrote the file"，极易误判为超时** | +| └ Impress（pptx） | ✅ 元包带入 | ❌ | ❌ | 114 上 pptx→pdf 100% 失败 | +| └ Writer（docx） | ✅ | ✅ | ❌ | — | +| pandoc | ✅ 手装 3.6.4 二进制（base.Dockerfile:22-27） | ⚠️ 待验证 | ❌ | docx 读取路径，镜像唯一开箱即用的一条 | +| pdftoppm (poppler-utils) | ❌ apt 清单无（⚠️ 是否被 libreoffice 依赖链顺带带入待镜像内实测） | ❌ 待验证 | ⚠️ 待验证 | 视觉 QA 断 | +| zip / unzip | ❌ python:3.11-slim 不自带、apt 清单无 | ⚠️ 待验证（RHEL 通常自带） | ⚠️ 待验证 | pptx/docx 编辑主干回包步断；可用 `helpers/rezip()` 绕开 | +| node | ❌ **两个 Dockerfile 全文 grep 零命中** | ✅ v22.17.1（手动装，非交付基线） | ⚠️ 模板带 node 但不承诺 | 官方交付形态下 pptx/docx 创建主干 100% 死 | +| npm | ❌ | ⚠️ 在但 `npm install` 静默失败 | ⚠️ | 自愈路径断 | +| pptxgenjs / docx-js / sharp | ❌ | ❌ | ❌ | 同上 | +| 中文字体 | ✅ fonts-wqy-zenhei | ⚠️ 只有 Noto CJK | ❌ 不承诺任何 CJK | `local_executor.py:106-109` 硬注入 `family="WenQuanYi Zen Hei"` 在 114 **命中不到字体**，只出 warning；两边 LO 渲染宽度不一致，视觉 QA 结论不可跨环境复用 | +| Liberation / Carlito / Caladea（metric 兼容替换） | ❌ `--no-install-recommends` 已丢掉 LO 的 Recommends | ❌ 待验证 | ❌ | pptx SKILL.md:129-134 整套字号/溢出 QA 的可信度前提 | +| gcc | ✅ build-essential | ✅ | ⚠️ | 无风险（走不到） | +| **技能包 `skills/` 子树本身** | ✅ local 模式物化到 cwd | ✅ | ❌ **结构性不可见** | 见下 | + +**E2B 那一列其实不必逐项看**：`_generate_tools`（拍 `os.walk` 快照建 copy-in 清单）在 `_create_agent`→`materialize_session_skills` 之前执行（task_exec.py:411→427 / 535→542 / 612→628），技能文件落盘时快照已拍完，**skills/ 永远进不了沙箱**；且 `e2b_executor.py:83` 用 `include_skills=False` 把 skills 路径提示从工具描述里刻意剥掉了。结果是最坏组合：模型能通过 `read_file` 读到 SKILL.md 正文（那条路走 WorkspaceBackend/MinIO，与沙箱无关），却拿不到说明书里的脚本。E2B 侧还有三条独立硬伤：单次执行超时回退 300s（local 是 600s，技能文案是按 600s 写的）、copy-in 单文件 5MB 硬阈值且超限静默丢弃、`_walk_sandbox` 只 `files.list('./')`（SDK 默认 depth=1）导致 `output/` 子目录产物不出现在返回的 `new_files` 里。 + +--- + +## 3. 短期方案：114 环境补齐清单 + +### 3.0 先定位后端 venv（这是必须先做的一步） + +代码执行器起子进程用的是 **`sys.executable`**（`local_executor.py:172`），即灵思 worker 自己的解释器。**装到系统 python 一律无效**。 + +```bash +# 找出 linsight worker 进程的真实解释器 +pgrep -af 'linsight|uvicorn|celery' +readlink -f /proc//exe # → /path/to/.venv/bin/python +export VENV_PY=/path/to/.venv/bin/python # 待验证：114 上的实际路径 +``` + +⚠️ 连带坑：模型如果在代码里写 `subprocess.run(["python", "scripts/recalc.py", ...])`，那个 `python` 走的是 **PATH**，未必是 venv。适配版技能必须要求用 `sys.executable`。同理 `infer_lang`（`local_executor.py:78-92`）把以 `python ` / `pip` 开头的整块代码判成 **sh 直接执行**，模型贴一行 `python scripts/recalc.py` 会走 shell 分支、解释器再次跑偏。 + +### (a) Python 包 — 必须装进后端 venv + +| 命令 | 解决什么 | +|---|---| +| `$VENV_PY -m pip install defusedxml`（或 `uv pip install --python "$VENV_PY" defusedxml`） | 让 pptx 的 `clean.py`/`thumbnail.py`、docx 的 `merge_runs.py`/`comment.py`、以及三家共用的 `office/validate.py` 不再在首行 ImportError。**性价比最高的一条** | +| `$VENV_PY -m pip install 'markitdown[pptx,xlsx]'` | 恢复 pptx 的读取路径与 Content QA（`markitdown output.pptx \| grep -iE ...` 占位符残留检查）、xlsx 的 quick look。⚠️ 其传递依赖体量**待验证**（不同版本 magika/onnxruntime 是否为可选 extra 有差异），装前先 `pip install --dry-run` 看清单 | + +已有无需装：openpyxl 3.1.5、pandas 2.3.3、Pillow 12.0.0、lxml 6.0.2、python-pptx 1.0.2、python-docx 1.2.0、pymupdf 1.26.6、pypandoc 1.15。 + +**离线获取**：在同架构联网机上 `pip download defusedxml 'markitdown[pptx,xlsx]' -d ./wheels --platform manylinux2014_x86_64 --python-version 311 --only-binary=:all:`，rsync 到 114 后 `$VENV_PY -m pip install --no-index --find-links ./wheels defusedxml markitdown`。defusedxml 是纯 python wheel，无平台问题。 + +### (b) 系统包 + +| 命令 | 解决什么 | +|---|---| +| `sudo dnf install -y libreoffice-calc` | **114 当前最硬的阻塞**：xlsx skill 的 `recalc.py` 主干（SKILL.md 标 mandatory）。没有 Calc 过滤器时 soffice 命令存在但加载不进 xlsx，报错伪装成超时 | +| `sudo dnf install -y libreoffice-impress` | pptx→pdf 转换，`thumbnail.py:191-198` 与视觉 QA 的前置 | +| `sudo dnf install -y poppler-utils` | 提供 `pdftoppm`，pptx/docx 视觉 QA 的唯一实现（bisheng 已有 pymupdf 可替代，但会打破 `slide-01.jpg` 零填充命名约定，SKILL.md 与 `thumbnail.py:215` 的 glob 都依赖它） | +| `sudo dnf install -y zip unzip` | pptx/docx「解包→改 XML→回包」主干的回包步（`zip -Xr` 的 `-X` 是刻意去 extra field） | +| `sudo dnf install -y liberation-fonts google-crosextra-carlito-fonts google-crosextra-caladea-fonts` | 给 SKILL.md 点名的 Arial/Times/Calibri/Cambria 做 metric-compatible 替换，否则视觉 QA 量出来的行宽/溢出结论不可信 | +| `sudo dnf install -y wqy-zenhei-fonts` | 让 `local_executor.py:106-109` 硬注入的 `family="WenQuanYi Zen Hei"` 真正命中，与官方镜像对齐；否则 matplotlib 中文出豆腐块且只有 warning | +| — | **ghostscript / ImageMagick 不用装**：三个 skill 零引用 | + +**离线获取**：联网同版本 RHEL 上 `dnf download --resolve --destdir=./rpms libreoffice-calc libreoffice-impress poppler-utils zip unzip liberation-fonts`，拷贝后 `sudo dnf install -y ./rpms/*.rpm --disablerepo=*`。注意 libreoffice-impress 的依赖闭包较大（会拉 core/common/ure/语言包），`--resolve` 必须带。 + +### (c) node 包 + +```bash +# 先确认 npm 静默失败的真因（几乎总是无外网 / 无 registry） +npm config get registry && npm ping +# 内网 registry 镜像写法 +npm config set registry https://registry.npmmirror.com # 或企业 Nexus/Verdaccio 地址 +npm config set strict-ssl false # 仅当内网 Nexus 用自签证书 + +npm install -g pptxgenjs # pptx「创建 deck」主干 +npm install -g docx # docx「创建文档」主干 +npm install -g sharp react react-dom react-icons # pptx 图标流水线（degraded） +export NODE_PATH=$(npm root -g) # 全局安装后 require() 才找得到，必须注入执行器进程环境 +``` + +**离线获取**：联网机 `npm pack pptxgenjs docx`（纯 JS，直接可搬）；`sharp` **不能这么搬**——它带 libvips 预编译原生二进制，必须 `npm install --os=linux --cpu=x64 --include=optional sharp` 后整棵 `node_modules` 打包搬运，且信创 arm64/龙芯平台可能根本没有对应预编译产物（**待验证**，届时需源码编译 libvips）。 + +**注意 114 上 npm 装完只对 114 有效，跟发布镜像毫无关系**——官方镜像连 node 运行时都没有。 + +--- + +## 4. 官方镜像要不要改（中长期） + +| 依赖 | 建议 | 理由 | 体积增量量级 | +|---|---|---|---| +| `defusedxml` → pyproject | **加** | 纯 python、无传递依赖、解锁三个 skill 的校验/清理脚本；同时它是 `python-pptx` 依赖链里**没有**的（uv.lock:5057-5062 已核） | < 100 KB | +| `poppler-utils` → base.Dockerfile apt | **加** | 视觉 QA 的唯一实现；也是通用能力（PDF 转图在别处也用得上） | ~10-15 MB（含依赖，量级估计，待实测） | +| `zip` `unzip` → apt | **加** | 成本近乎零，且 OOXML 手术是三个 skill 的通用路径；顺带解决其它 skill 里裸 `unzip` 的失败 | < 1 MB | +| `fonts-liberation` `fonts-crosextra-carlito` `fonts-crosextra-caladea` → apt | **加** | 现在 `--no-install-recommends` 把 LO 的 Recommends 字体全丢了，这三个是 Arial/Calibri/Cambria 的 metric 兼容替换，直接决定 LO 渲染保真度与 QA 可信度 | ~10-12 MB | +| `markitdown[pptx]` → pyproject | **暂缓，先实测依赖闭包** | 功能上可被 python-pptx/openpyxl/pymupdf 自造替代（代价是输出格式不是 ``，SKILL.md 的 grep 检查会连带失效）。若其传递依赖拉进 onnxruntime 级别的东西，收益不抵体积 | 待验证 | +| **node + npm + pptxgenjs/docx-js/sharp** | **不加** | ① node20 runtime 本身 ~120 MB+；② sharp 的 libvips 预编译二进制按平台分发，信创 arm64/龙芯无保证；③ 引入第二套包管理器和第二条供应链，私有化离线交付的镜像审计成本陡增；④ **收益可被 python-pptx/python-docx 完全替代**（bisheng 已有），代价只是放弃官方 SKILL.md 的 API 指导——而这份指导本来就要重写（见 §5） | +150 MB 起，且不可控 | +| ghostscript / ImageMagick / pdftk | **不加** | 三个 skill 零引用 | +| E2B 自建模板 | **走沙箱路线才做** | 现状 `Sandbox()` 调用处（`e2b_executor.py:88`）没传 `template=`，走 SDK 默认 `code-interpreter-v1`；模板 id 在前端表单、`extra` schema、executor 构造函数三处都无处可配。要走沙箱需同时改这三处 + 自建预装 libreoffice/pandoc/CJK 字体的模板 + 把 skills 物化提到 `_generate_tools` 之前 | 独立工作项 | + +合计 apt 侧增量约 **20-30 MB**（量级估计，需实测），相对当前 base 镜像（含 libreoffice 元包 + playwright chromium，GB 级）可忽略。 + +--- + +## 5. 直接用官方 skill 的可行性判断 + +### 结论：**(c) 做适配版**，且这是唯一现实选项。 + +理由不是依赖装不齐——依赖按 §3/§4 补完后，xlsx 主干在官方镜像上**本来就能跑**、docx 只死"创建"一半。真正的判死点是下面这批**跟依赖无关的、对 Claude 特有能力的假设**，装再多包也不会消失： + +| 假设 | 出处 | 对 Qwen/DeepSeek 的后果 | 能否靠装依赖解决 | +|---|---|---|---| +| **API 知识靠模型脑内**：pptx「The model knows the API; these are the footguns」，docx 同款，xlsx 一行 openpyxl 示例都不给 | pptx SKILL.md:31 / docx:21 / xlsx:11 | 整个创建主干 = 模型现场手写 JS/py，只给 20 条"别踩的坑"不给 API。弱模型写出的错误恰恰**不在**那 20 条里，`validate.py` 只查 OOXML 不查 JS，抓不住 | 否 | +| **视觉 QA 靠看图**：pptx 整节 12 类目视缺陷 + `Pass the absolute paths directly to the view tool`；docx `ls page-*.jpg # then Read the images` | pptx:204-232 / docx:42 | ① 硬编码了 agent 侧工具名 `view`/`Read`，灵思没有同名工具；② 灵思 qwen3.5 走 dashscope，**已知不收 base64 图片**（180 html-ppt 的教训）。这一整节等于空转 | 否 | +| **负向清单靠自律**：xlsx 要求恰好六个函数写 `_xlfn.` 前缀、禁用 XLOOKUP/XMATCH/SORT/FILTER/UNIQUE/SEQUENCE，**同行亲口承认这种情况 `recalc.py` 报 `total_errors: 0`** | xlsx:67-68 | Qwen/DeepSeek 生成 Excel 公式默认吐 XLOOKUP 概率很高，校验器抓不住 → 交付一个在 Excel 里全是 `#NAME?` 的文件 | 否 | +| **长上下文同时持有 ~35 条互相牵制的设计禁令**：pptx 85 行散文规则含 14 条 Avoid（两条全大写 NEVER）；xlsx 要求写每个 cell 时同时满足"用公式不硬编码/百分比存小数/蓝字硬编码黑字公式绿字跨表红字跨文件" | pptx:80-164 / xlsx:80-95 | 指令跟随弱的模型生成到第 8 张 slide 就漂回默认样式 | 否 | +| **裸 shell 视角**：`markitdown deck.pptx`、`(cd unpacked && zip -Xr ../out.pptx .)`、`pdftoppm -jpeg -r 150` | 三家通篇 | `bisheng_code_interpreter` 执行的是 **Python**，模型必须自己包一层 subprocess；弱模型经常把 shell 命令当 Python 直接写。且 `infer_lang` 会把 `python ...` 开头的整块判成 sh，行为再次分叉 | 否 | +| **相对路径 + 特定 sys.path 布局**：「Script paths are relative to this skill's directory」，且 `scripts/office/` 无 `__init__.py`（PEP 420），xlsx/pptx 的 `from office.helpers import` 要求 `scripts/` 在 sys.path，而 `office/validate.py` 的裸 `from helpers import` 要求 `scripts/office/` 在 sys.path——**两套导入根互不兼容** | xlsx:18/37, docx:17, pptx:19 | 灵思 cwd = 任务工作区、skill 物化在 `skills//`，原样执行必然 `No such file`；改写成 `python -m` 或拍平目录必 ImportError | 否 | +| **原地精确编辑工具**：docx「edit `unpacked/word/document.xml` **in place** — do NOT reformat or pretty-print」，隐含 Claude Code 的 `Edit`（str_replace） | docx:55 | 灵思是 `write_file` 全量覆盖，且**已知大参数被截断触发死循环**；照做会把 document.xml 整体重写并顺手 pretty-print，正中明令禁止 | 否 | +| **多轮工具循环 + 自我纠错**：xlsx 的"先写 2-3 个公式验证 → 再铺开 → 重算 → 修"四段式；pptx「a subagent works well for this」 | xlsx:45-50 / pptx:206 | 灵思子代理 **30 轮预算跨 task 调用共享**（已知坑），逐页图审会把预算烧光 | 否 | +| **clean exit ≠ clean 产物的元认知**：`errors_found` 也 exit 0，只有 `error` 键才非零退出 | xlsx:44-48 | 弱模型看到 exit 0 就宣布完成 | 否 | +| **假设有外网可 pip/npm install**：三家都写「Only if an import fails: `pip install` / `npm install`」 | xlsx:16,99 / pptx:31,50 / docx:21 | 私有化内网下变成几十秒 pip 超时空转，而不是快速失败换路径；且 `local_executor` 描述里明说"do NOT run pip install"，两条指令直接打架 | 否 | +| **身份烙印**：`comment.py:321` `--author` 默认 `"Claude"` | docx | bisheng 产出的 Word 批注作者一律显示 Claude | 改代码即可（属适配工作） | + +外加两个**与 bisheng 执行器正面相撞**的工程点，同样必须在适配层解决： + +1. **失败时 stdout 被整段丢弃**（`local_executor.py:205-215`：`if proc.returncode: logs = stderr` 否则 `logs = stdout`，二选一）。而这些 skill 的诊断信息几乎全走 stdout——`recalc.py:303-304` 失败 JSON 走 stdout + exit 1，`validate.py:169` 失败 exit 1 而所有 FAILED 明细都是 `print`。结果：模型拿到一片空白 stderr，pptx SKILL.md:202 承诺的「Every failure names its fix」直接归零。 +2. **失败即丢产物**（`local_executor.py:340-341/394-406`：exitcode != 0 直接 return，`file_list` 为空）。前半段已写好 `output/x.pptx`、最后一行报错，产物一律不回传。 + +### 三选一的判据 + +- **(a) 原样导入**：排除。执行期必炸（defusedxml/node/markitdown），且 LICENSE.txt 明文禁止「Extract these materials from the Services or retain copies outside the Services / Reproduce or copy / Create derivative works」，三份 LICENSE md5 相同（f8515c36…），frontmatter 均为 `license: Proprietary`。**bisheng 是开源仓 + 商业私有化交付，把这套文件打成技能包分发（尤其提交进公开仓）是合规红线**，不是技术问题但会卡发布。 +- **(b) 装依赖后原样导入**：排除。上表 11 条假设无一能靠装包解决；node 那条在官方镜像上还额外不可解。 +- **(c) 适配版**：**已被验证有效**。仓库里的 `linsight-skills/bisheng-pptx/` 就是范例——**实测源码 70,009 B（7 个文件，不含 `__pycache__`）对比官方 pptx 的 1,139,175 B**，体积 1/16，且把 pptxgenjs 换成 python-pptx、把 view 工具换成脚本自检、把路径改成工作区语义、附带 `references/pptx-cookbook.md` 与 `design-zh.md` 两份中文参考（正是官方缺的那份 API 速查）。xlsx 与 docx 照此复刻即可。 + +**适配版的最小改造清单**（三家共通）：① 主干路径改 python（xlsx 本来就是；pptx→python-pptx；docx→python-docx），连带**整段替换** SKILL.md 里的 footgun 章节，否则模型会拿 pptxgenjs/docx-js 的 API 名去调 python 库；② 所有脚本路径改成基于 `skills//` 的绝对路径调用，并统一 sys.path 布局；③ 诊断结论改写 stderr（或反过来改执行器保留 stdout）；④ 脚本一律 `exit 0`（bisheng 技能包约定）；⑤ 删掉视觉 QA 章节，换成脚本化的结构自检（`inspect_deck.py` 那种）；⑥ 删掉 `pip install` / `npm install` 的自愈指引，改成"换已装库"；⑦ `accept_changes.py` 的 `/tmp/libreoffice_docx_profile` 硬编码单例改带 uuid 的临时目录（照抄同目录 `soffice.py:41-45`），并把 `:76-80` 那个「TimeoutExpired 返回 Successfully accepted」的**超时伪装成功**改成如实报错——否则多任务并发会静默产出未接受修订的文件；⑧ `soffice.py:50` 的 `$TMPDIR/lo_socket_shim.so` 固定文件名同样是并发不安全点（虽然容器内走不到）。 + +--- + +## 6. 技能包体积与 bisheng 限制的冲突 + +| 项 | 字节 | 占比 | 对照限制 | +|---|---|---|---| +| xlsx 全目录 | 1,102,893 B (~1.05 MB) | — | zip ≤ 10 MB ✅ / 解包 ≤ 100 MB ✅ | +| pptx 全目录 | 1,139,175 B (~1.09 MB)，**实测 zip 后 169,845 B** | — | ✅ 余量两个数量级 | +| docx 全目录 | 1,128,695 B (~1.08 MB)，61 个文件 | — | ✅ | +| └ 其中 `office/schemas/` | 976,913 B | xlsx 88.6% / docx 86.5% / pptx 85.8% | 三份互为重复 | + +**结论：体积不是约束，压根不用裁也能装。** 但三家同时上架会有 3 × 954 KB 的 XSD 重复占用解包空间（约 2.9 MB），值得留意的是**它在 xlsx 上是 100% 死重**——`validate.py:142-148` 对 xlsx 家族在构造任何 validator 之前就 `sys.exit(0)`，连 `base.py:58` 映射的 `sml.xsd`（242,277 B，全包最大单文件）都永远不会被加载。 + +裁剪建议（若走原样打包路线才需要，走适配版则自然不存在）： + +- **xlsx**：删 `scripts/office/schemas/`（976,913 B）+ `scripts/office/validators/`（~88 KB）+ `scripts/office/helpers/`（~20 KB），只留 `scripts/recalc.py` + `scripts/office/soffice.py`。剩 ~126 KB，**主干路径零功能损失**。 +- **pptx**：可删 `sml.xsd`(242 KB) + `wml.xsd`(171 KB) + `schemas/microsoft/`(52 KB) ≈ 465 KB。前提是先实测 `pml.xsd` 的 `@import` 链（已核实只 import `dml-main.xsd` / `shared-commonSimpleTypes.xsd` / `shared-relationshipReference.xsd`，不碰 sml/wml）。 +- **docx**：可删 `sml.xsd` + `pml.xsd` + `dml-chart.xsd` + `dml-diagram.xsd` 等约 460 KB，以及 `helpers/pptx_*.py`（~11 KB）。**但 XSD 之间有 `xs:import` 交叉引用**（`base.py:21` 用 base_url 解析），删前必须实测 `wml.xsd` 加载链，别硬删。 +- **三家同上架**：`office/soffice.py` / `office/validate.py` / `office/validators/base.py` / `office/helpers/__init__.py` 四文件 md5 完全相同，可抽成一份共享，省 ~1 MB × 2。 + +⚠️ 两个删不得的红线：① `base.py:111` 的 `schemas_dir = Path(__file__).parent.parent / "schemas"` 是相对 `__file__` 解析的，**只拷 `scripts/*.py` 不带 `schemas/` 会让 `_load_schema` 抛异常，而异常被 `base.py:809-810` 吞成一条 error 字符串，表现为"所有文件都有新错误"的假阳性洪水**；② 技能包 name 必须匹配 `^[a-z0-9]+(-[a-z0-9]+)*$`，非法名现在会被 `slugify_pinyin` 自动改写并原地重写 SKILL.md 的 frontmatter（deepagents 硬约束：frontmatter name 必须等于目录名）。 + +**待验证清单**（成本极低，建议上线前在容器/114 各跑一次一次性确权）： + +```bash +which pdftoppm zip unzip node npm markitdown soffice pandoc gs +soffice --headless --convert-to pdf /tmp/probe.pptx # 探 Impress +soffice --headless --convert-to pdf /tmp/probe.xlsx # 探 Calc +$VENV_PY -c "import defusedxml, lxml, openpyxl, pptx, docx; print('ok')" +fc-list | grep -iE 'liberation|carlito|caladea|wqy|noto.*cjk' +``` + +其余待验证项：114 上 pandoc 是否存在及版本；`markitdown[pptx,xlsx]` 的传递依赖闭包；`poppler-utils`/`zip`/`unzip` 是否被官方镜像的 libreoffice 元包或 `playwright install-deps` 间接带入；信创 arm64/龙芯平台 sharp 的 libvips 预编译可得性；E2B `code-interpreter-v1` 模板的实际库清单（Anthropic 从未文档化该运行时的 node/LibreOffice/pandoc/markitdown 版本，属结构性盲区，只能实测反推）。 diff --git "a/docs/PRD/2.6 \347\201\265\346\200\235 deepagents \350\277\201\347\247\273 PRD/\347\201\265\346\200\235\344\273\243\347\240\201\346\211\247\350\241\214\346\262\231\347\256\261\351\200\211\345\236\213\350\260\203\347\240\224\357\274\210\347\247\201\346\234\211\345\214\226\345\234\272\346\231\257\357\274\211.md" "b/docs/PRD/2.6 \347\201\265\346\200\235 deepagents \350\277\201\347\247\273 PRD/\347\201\265\346\200\235\344\273\243\347\240\201\346\211\247\350\241\214\346\262\231\347\256\261\351\200\211\345\236\213\350\260\203\347\240\224\357\274\210\347\247\201\346\234\211\345\214\226\345\234\272\346\231\257\357\274\211.md" new file mode 100644 index 0000000000..9314676cfb --- /dev/null +++ "b/docs/PRD/2.6 \347\201\265\346\200\235 deepagents \350\277\201\347\247\273 PRD/\347\201\265\346\200\235\344\273\243\347\240\201\346\211\247\350\241\214\346\262\231\347\256\261\351\200\211\345\236\213\350\260\203\347\240\224\357\274\210\347\247\201\346\234\211\345\214\226\345\234\272\346\231\257\357\274\211.md" @@ -0,0 +1,335 @@ +# 灵思代码执行沙箱选型调研（私有化场景）：E2B 是否仍是最优解 + +> **调研日期**：2026-08-11 +> **问题**：bisheng 作为开源企业级 LLM 应用平台、以**私有化部署**为主要交付形态，当下 E2B 沙箱还是不是最优选型？deepagents 官方是否推荐过开源沙箱？ +> **方法**：多 agent 并行 deep-research（一手来源：GitHub 仓库文件 / 官方文档站 / release notes / PR & Issue 状态）+ 5 条高风险主张对抗式核验（默认判死，须一手证据才翻案） +> **姊妹篇**：[Anthropic 官方 office skills 依赖审计与 bisheng 环境缺口](./Anthropic%20office%20skills%20依赖审计与%20bisheng%20环境缺口.md) + +**一句话结论**：**E2B 已不是最优选型，且这是架构性结论而非配置/版本问题** —— 官方自托管硬绑 Cloudflare + Terraform/Nomad + 嵌套虚拟化，单机版 PR 已夭折，云版只有 US/EU 两区导致数据出境。推荐落点是**「独立容器 + compose 原生加固」**（自建 `bisheng-code-runner` 或直接用 llm-sandbox），对隔离有硬要求时叠加 gVisor。但**比换沙箱更紧急的是 local 模式的三个一手实锤缺陷**（Fernet 密钥硬编码在开源代码里、子进程全量继承 MinIO 凭据、compose 显式 `seccomp:unconfined` 且无 cgroup 限额）。 + +--- + +## 1. 结论先行 + +**E2B 已经不是 bisheng 的最优选型，而且这是架构性结论，不是版本或配置问题。** 官方自托管在 [infra README](https://github.com/e2b-dev/infra/blob/main/README.md) 里把「General linux machine」写成未勾选的空框，[self-host.md](https://raw.githubusercontent.com/e2b-dev/infra/main/self-host.md) 硬性要求 Cloudflare 账号 + 托管在 Cloudflare 的域名 + PostgreSQL + Terraform/Nomad/Consul + 嵌套虚拟化，官方自己做的单机版 [PR #1837「e2b-lite」已于 2026-04-26 关闭未合并](https://github.com/e2b-dev/infra/pull/1837)，替换 Cloudflare 的 [Route53 PR #2392 也已关闭未合并](https://github.com/e2b-dev/infra/pull/2392)。云版只有 US/EU 两个区域，代码与文件全量出境，中国私有化客户直接出局。 + +**推荐落点是「独立容器 + compose 原生加固」这条容器派路线**——要么自建 `bisheng-code-runner`，要么直接用 [llm-sandbox](https://github.com/vndee/llm-sandbox)（MIT，1104★，双向 `copy_to_runtime`/`copy_from_runtime`，官方支持 air-gapped）。它是唯一同时满足「离线可部署 + 不要嵌套虚拟化 + 能装 libreoffice/pandoc/node + docker-compose 交付 + x86/arm64/龙芯通吃」的组合，代价是默认共享宿主内核、不是 VM 级安全边界。**这条路线最大的技术风险已于 2026-08-11 实测排除**：整条 office 产出链路（soffice 转 docx/xlsx/pptx→PDF、pandoc、PyMuPDF、4 路并发、中文渲染）在 `cap_drop:ALL` + `no-new-privileges` + 非 root + 只读 rootfs + `/tmp` noexec + `network:none` + cgroup 限额的最严档下**八档全绿**，详见 [§8 POC 记录](#8-poc-记录受限容器下的-office-链路2026-08-11-实测)。 + +**所有 microVM 派全部出局**：E2B、[microsandbox](https://github.com/superradcompany/microsandbox)（README 明写 Linux 要 KVM，且自认 beta）、[Kata](https://github.com/kata-containers/kata-containers/blob/main/docs/install/README.md)（首句即 "requires nested virtualization or bare metal"）、Firecracker 都硬依赖 `/dev/kvm`；而 Broadcom [KB 313547](https://knowledge.broadcom.com/external/article/313547/support-for-running-esxi-as-a-nested-vir.html) 明确「不支持在 vSphere 虚机内运行嵌套 hypervisor」——在 VMware 客户处这不是「默认关闭」，是合同层面不受支持。**2026-08-11 产品侧补充的口径把这条钉死了**：交付以**虚机**为主、基本无公有云云主机 —— 虚机里开 microVM 必须由客户虚拟化团队逐台开启嵌套虚拟化，**属于「要客户改配置」的实施阻力，因此 microVM 不进默认交付形态**，最多作为物理机客户的可选升级档。**Daytona 也别碰**：[官方 2026-06-11 宣布转闭源](https://www.daytona.io/dotfiles/updates/daytona-is-going-closed-source)，main 分支源码已清空只剩 README，末版 v0.190.0 的许可证是 [AGPL-3.0](https://raw.githubusercontent.com/daytonaio/daytona/v0.190.0/LICENSE)，7.2 万 star 是历史存量。 + +**但真正紧急的不是换沙箱，而是 local 模式的三个一手实锤缺陷**：Fernet 密钥硬编码在开源代码里（`src/backend/bisheng/core/config/settings.py:20`）、`subprocess.Popen` 未传 `env=` 导致 MinIO AK/SK 全量继承、容器以 root 运行且 compose 显式写了 `security_opt: seccomp:unconfined` 并且没有任何 cgroup 限额。这三条不换沙箱也必须先修，且优先级高于选型本身。 + +--- + +## 2. 评估维度与权重 + +私有化 ≠ SaaS，权重必须重排：SaaS 选型里排第一的「隔离强度 / 冷启动性能 / 生态热度」，在私有化里全部让位于「在客户那台机器上到底能不能跑起来」。 + +| 维度 | 权重 | 为什么这样加权 | +|---|---:|---| +| **离线/内网可部署性** | 20 | 交付形态是 docker-compose 进客户内网，常常无外网。任何需要联网拉 kernel、拉 CDN、连 Cloudflare、回连厂商控制面的方案，在实施现场就是 0 分。这是 **gate**：得 0 即整体否决。 | +| **是否要求嵌套虚拟化** | 15 | 同为 gate。Broadcom 明确不支持 vSphere guest 内跑第三方 hypervisor（KB 313547），且配了 vGPU/PCI 直通的 VM 连勾选框都点不了（[KB 437651](https://knowledge.broadcom.com/external/article/437651/unable-to-enable-expose-hardware-assiste.html)）。这一条单独拆出来，因为它把整个 microVM 派一次性砍掉，而它与「离线」是正交的。 | +| **自定义镜像能力** | 12 | 灵思的价值链是产出 docx/pptx/xlsx/pdf，依赖 libreoffice(含 impress) + pandoc + node + 中文字体 + pymupdf。装不了原生二进制的方案（WASM/Pyodide/Riza）在这里直接归零，不是扣分。 | +| **运维复杂度** | 12 | 横向锚点：Dify / RAGFlow / FastGPT / Coze Studio / Open WebUI 的自托管主路径全是 docker-compose，只有 E2B 走 Terraform+Nomad+Consul。合理上限就是「compose 一把梭」，至多外加「宿主机装一个二进制」。
**2026-08-11 修正**：产品侧反馈**部分交付确实会上 K8s**，故本维度的判据从「必须是 compose」改为「**compose 与 K8s 两种编排下都能跑**」——**强制要求客户上 K8s** 才扣分，**支持 K8s** 不扣分。据此重评了下表的 K8s 原生方案（原「否决·形态错配」不再成立）。 | +| **arm64/信创支持** | 10 | 鲲鹏/飞腾=aarch64、海光/兆芯=x86_64、龙芯=loongarch64。gVisor 与 Firecracker 官方支持列表都只有 x86_64/ARM64，**龙芯全场无解**——只有普通容器路线能覆盖。 | +| **隔离强度** | 10 | 权重刻意压低。理由：当前 local 模式的隔离基线是 **0**（root + seccomp unconfined + 同网络直连中间件 + 无 cgroup），任何容器化都是数量级改善；而 [NIST SP 800-190](https://nvlpubs.nist.gov/nistpubs/specialpublications/nist.sp.800-190.pdf) 也明说容器不是 VM 级边界——在「跑不起来」面前，隔离是二阶问题。 | +| **许可证可商用性** | 8 | bisheng 有商业版。AGPL-3.0（Daytona）、GPL-3.0（Judge0）会卡国企/金融客户的开源合规审查；还要警惕「代码 Apache-2.0 但运行时绑死平台」的伪开源（Cloudflare Sandbox SDK）。 | +| **Python SDK + 文件 copy-in/out** | 6 | 权重不高，因为 bisheng 的 `BaseExecutor` 抽象极薄，SDK 缺失可以自己补；但**文件 copy-in 的时机可控性**是刚需（见 §4 技能包问题）。 | +| **迁移成本（对照 BaseExecutor）** | 4 | 实测契约只有：`__init__(minio, **kwargs)` + 一个 `@abstractmethod run(code)` + 可选 `close()`，另加一个隐式的 `description` property（`load_tools.py:112` 用到但基类没声明）。换沙箱的代码量下限很低，所以这项拉不开差距。 | +| **社区成熟度** | 3 | 权重最低。理由：私有化交付一旦发版就冻结，客户环境不跟上游滚版本；star 数与 release 频率对我们的实际价值远低于 SaaS 场景。反面教材是 Daytona——7.2 万 star 照样停维护。 | + +**判据规则**：离线=0 或 嵌套虚拟化=0 或 自定义镜像=0，任一命中即**否决**，总分仅作参考不作排序依据。 + +--- + +## 3. 候选对比大表 + +评分 0–5，总分 = Σ(score/5 × 权重)，满分 100。⛔ = 命中否决门。 + +| 候选 | 离线20 | 无需嵌套虚拟化15 | 自定义镜像12 | 运维12 | 信创10 | 隔离10 | 许可证8 | SDK/文件6 | 迁移4 | 成熟3 | **总分** | 判定 | +|---|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|--:|---| +| **E2B 云（基准）** | ⛔0 | – | 4 | 5 | 0 | 5 | 0 闭源SaaS | 5 | 5 | 5 | 59.6 | **否决**·数据出境 | +| **E2B 自托管** | ⛔0 | ⛔0 | 5 | 0 | 1 | 5 | 5 Apache-2.0 | 5 | 5 | 4 | 44.4 | **否决**·结构性 | +| **自建加固容器（bisheng-code-runner）** | 5 | 5 | 5 | 4 | 5 | 2 | 5 自有 | 3 | 3 | 5 | **87.6** | ✅ 推荐 | +| **llm-sandbox** | 5 | 5 | 5 | 4 | 3 | 2 | 5 MIT | 5 | 4 | 2 | **85.0** | ✅ 推荐（现成实现） | +| **上述二者叠加 gVisor(runsc)** | 4 | 5 | 5 | 3 | 2 | 4 | 5 Apache-2.0 | 3 | 3 | 5 | **79.2** | ⭕ 可选加固档 | +| **OpenSandbox（阿里）** | 3 | 5 | 5 | 3 | 2 | 3 可插拔 | 5 Apache-2.0 | 5 | 3 | 3 | **74.4** | ⭕ 观察 | +| **microsandbox** | 3 | ⛔0 | 5 | 4 | 2 | 5 | 5 Apache-2.0 | 5 | 3 | 2 | 65.2 | **否决**·KVM | +| **K8s agent-sandbox / E2B 兼容层** | 2 | 4 | 5 | 3 | 3 | 4 | 5 Apache-2.0 | 4 | 5 | 2 | **71.2** | ⭕ 已解除否决·仅当客户已有 K8s | +| **Kata Containers** | 4 | ⛔0 | 5 | 1 | 4 | 5 | 5 Apache-2.0 | 1 | 1 | 5 | 61.4 | **否决**·裸金属专供 | +| **Firecracker 自建** | 4 | ⛔0 | ⛔1 | 0 | 2 | 5 | 5 Apache-2.0 | 0 | 0 | 5 | 43.4 | **否决** | +| **Daytona** | – | – | – | – | – | – | ⛔0 AGPL-3.0+已闭源 | – | – | – | – | **否决**·2026-06 停维护 | +| **WASM / Pyodide / Riza** | 3 | 5 | ⛔0 | 4 | 5 | 5 | 混合 | 2 | 2 | 3 | – | **否决**·跑不了原生二进制 | +| **Judge0** | 4 | 5 | ⛔1 | 1 | 2 | 2 | ⛔1 GPL-3.0 | 1 | 1 | 4 | – | 否决 | +| **Piston** | 3 | 5 | ⛔0 | 2 | 2 | 2 | 5 MIT | 1 | 1 | 4 | – | 否决·无自定义镜像/无产物取回 | +| **dify-sandbox** | 3 | 5 | 2 | 4 | 3 | 3 | 5 Apache-2.0 | 2 | 2 | 4 | – | 仅作参考实现 | +| **LangSmith Sandbox** | 2 | ⛔0 | 4 | ⛔0 K8s | 1 | 5 | 商业授权 | 5 | 4 | 4 | – | 否决 | +| **Modal / ACA Dynamic Sessions / Cloudflare** | ⛔0 | – | 4–5 | 5 | 0 | 4–5 | 闭源/平台绑定 | 3–5 | – | 5 | – | 否决·仅作能力对照 | + +**许可证坑单独点名**： +- **AGPL-3.0** — Daytona（v0.190.0 tag 的 LICENSE 首行为 "GNU AFFERO GENERAL PUBLIC LICENSE Version 3"；main 分支 GitHub API `license=null` 只是因为 LICENSE 随源码被删）。 +- **GPL-3.0** — Judge0，商业版分发需法务预判。 +- **LGPL-2.0+** — bubblewrap（与其余 Apache 候选不同）。 +- **平台绑定型伪开源** — Cloudflare Sandbox SDK 的 `packages/sandbox/LICENSE` 确是标准 Apache-2.0 且无平台限制条款，但运行时只能跑在 Cloudflare Workers+Containers 上，**「Apache-2.0」在这里不等于可私有化**。 +- **元数据不一致** — `e2b-code-interpreter` 的 PyPI 元数据标 MIT，仓库 LICENSE 实为 Apache-2.0；`langchain-sandbox` GitHub API `license=null` 而 README 标 MIT。选型必须核 LICENSE 文件本体，不能只看 badge。 +- **非开源** — Anthropic 官方 docx/xlsx/pptx skill 的 `LICENSE.txt` 三份完全相同（md5 `f8515c3694eb11622110ca76c7c15d1b`），条款禁止 "Extract these materials from the Services or retain copies outside the Services / Reproduce or copy / Create derivative works"——**不能原样打成 bisheng 技能包分发**，尤其不能提交进公开仓。 + +--- + +## 4. E2B 到底还是不是最优选型 + +**不是。** 在 bisheng 私有化场景下有五个致命问题，前三个是选型级红线： + +**（1）离线自托管前提不成立——这是架构问题不是配置问题。** [infra README](https://github.com/e2b-dev/infra/blob/main/README.md) 的 Supported cloud providers 至今是 `🟢 GCP / 🟢 AWS (Beta) / [ ] Azure / [ ] General linux machine`；当前 main 分支 `iac/` 下只有 `provider-aws` 与 `provider-gcp`。[self-host.md](https://raw.githubusercontent.com/e2b-dev/infra/main/self-host.md) 的 Prerequisites 硬性要求 **Cloudflare 账号 + 托管在 Cloudflare 的域名 + PostgreSQL**，工具链要 Packer / Terraform v1.7.5 / Golang / Docker Buildx / NPM，编排用 Nomad+Consul（不是 compose、不是 Helm），部署步骤里的 `make copy-public-builds` 还要从 E2B 公共对象存储拉 kernel/Firecracker/busybox。 + +尤其要注意三条**曾被寄望、但已经被证伪**的翻案路径（这三条前序调研遗漏，是最容易被人拿来反驳的点）： +- 官方做过单机/裸金属安装版 [PR #1837「e2b-lite」](https://github.com/e2b-dev/infra/pull/1837)，body 自述 "easily installable version of E2B infrastructure locally, or on bare-metal Linux machine"——**2026-04-26 关闭，未合并**。 +- [Issue #2211](https://github.com/e2b-dev/infra/issues/2211)（单节点部署诉求）曾以「e2b-lite 解决了这个场景」为由结案，但它指向的 PR 随后夭折，**答案落空**。E2B 成员 jakubno 在同 issue 明确表态 "We only support running it with Firecracker"，并称本地 compose 路径 "meant for local dev only, it isn't considered production ready"。 +- 仓库里现存的 `DEV-LOCAL.md` 是唯一不走 Terraform/Cloudflare 的路径，但开篇即 "This is a work in progress. Not everything will function as expected."，要求裸金属或带嵌套虚拟化的 VM + KVM，且**仍依赖 Google Cloud SDK**、使用硬编码测试凭据。 + +社区提交的裸金属 provider [PR #1882](https://github.com/e2b-dev/infra/pull/1882)（2026-07-09 关闭未合并）与替换 Cloudflare 的 [PR #2392](https://github.com/e2b-dev/infra/pull/2392)（2026-07-30 关闭未合并）同样失败。上游态度是「未支持」，不是「即将支持」。**许可证不是障碍**——三仓均为 Apache-2.0，无 BSL/SSPL；工程前提才是。 + +**（2）数据出境。** E2B 云只有 US 与 EU 两个区域（[status 页](https://status.e2b.dev/)），无中国区、无亚太区。所有代码片段、文件读写、环境变量全部出境，受《[促进和规范数据跨境流动规定](https://www.cac.gov.cn/2024-03/22/c_1712776611775634.htm)》（2024-03-22 网信办公布）与《数据出境安全评估办法》约束，豁免情形限于「不包含个人信息或者重要数据」的国际贸易/学术合作等场景——LLM 应用把用户文件送到境外沙箱不在豁免内。**e2b 模式在客户现场必须默认关闭。** + +**（3）模板构建联网——这条需要修正前序调研的过度表述。** 准确说法是：**SaaS 用法必须走 E2B 云构建**（V1 本地构建体系已于 [2026-08-01 随 access token 停用而失效](https://e2b.dev/docs/migration/v1-build-deprecation)）；但**自托管集群由自己的 Build 节点池（template manager）构建**，`make prep-cluster` 就在本集群 build base template，不必回连 E2B 云。所以「构建必须上云」只对 SaaS 成立。这一点无法挽救整体结论——因为承载它的集群本身就跑不了离线、离不开公有云、离不开 Terraform，且基础镜像仍需可达的 registry。 + +**（4）默认模板并不解决 bisheng 的痛点。** 拿到 [template.py 原文](https://raw.githubusercontent.com/e2b-dev/code-interpreter/main/template/template.py)：base 是 `python:3.13`，装了 **Node.js 20**（bisheng 镜像反而没有）、R 4.5、Java 11、`fonts-noto-cjk`；[requirements.txt](https://raw.githubusercontent.com/e2b-dev/code-interpreter/main/template/requirements.txt) 有 pandas 2.2.3 / matplotlib 3.10.9 / openpyxl 3.1.5 / python-docx 1.1.2 / opencv。但**没有 python-pptx、没有 libreoffice、没有 pandoc、没有 pymupdf**——办公文档转换链照样要自建模板。而且模板装的是 `fonts-noto-cjk`，`local_executor.py` 硬注入的 `matplotlib.rc("font", family="WenQuanYi Zen Hei")` 在 E2B 上会**原样复发中文方框**，和 114 RHEL 上是同一个 bug。 + +**（5）仓库里的既有实现缺陷，换沙箱也不会自动消失。** 这些是逐行核实的一手结论： + +- **技能包在 E2B 模式不可见 = 时序问题**。`workbench_impl.py:1496-1509` 用 `os.walk(file_dir)` 一次性拍快照生成 `file_list`；而该函数由 `task_exec.py:805-812` 的 `_generate_tools` 调用，skills 的物化在 `task_exec.py:827/843-845` 的 `_create_agent → materialize_session_skills`。三条驱动路径（fresh 411→427、resume 535→542、continue 612→628）**顺序都是先 tools 后 agent**，所以 `skills/` 落盘时快照已经拍完。代码里有显式自认（`e2b_executor.py:81-83` 的 `include_skills=False` 注释），测试 `test_e2b_description_omits_skills` 还把这行为钉死了。 +- **`required_files` 是死代码**。E2B 的 `run(code, required_files=None)` 签名存在，但工具 `args_schema` 只有 `python_code` 一个字段（`tool.py:47-58`），`_run` 也只传 `python_code`——模型**物理上无法**声明 required_files。 +- **运行期 copy-in 是空转**。`_materialize_working_set()` 永远 `return {}`（注释 `TODO(Wave 2)`），跨轮/中途新产生的工作区文件不会补推进沙箱。 +- **`workspace_backend` 在生产从没被注入**，`_copy_out` 里的 `workspace_backend.write` 恒为 no-op。 +- **`_walk_sandbox` 只看一层**：`self.sandbox.files.list('./')` 在 SDK 里默认 `depth=1`，所以 `new_files` 列不出 `output/report.pptx` 这类子目录产物；单测之所以绿，是因为 `FakeFiles.list` 无视 path 参数返回全部条目。文件还是能靠手写递归的 `sync_files_to_local` 回灌本地，但**模型拿不到产物清单**——`new_files` 这个正式契约名存实亡。 +- **单次执行超时只有 local 的一半**：`run_code(timeout=)` 从没被传，SDK 回退 `DEFAULT_TIMEOUT = 300s`；而 `LocalExecutor.DEFAULT_TIMEOUT = 600`。传给 `Sandbox(timeout=3600)` 的是**沙箱存活时长**不是执行超时，且 3600 正好卡在 E2B Hobby 档 1 小时上限。 +- **SDK 已严重滞后**：仓库 pin `e2b-code-interpreter==1.5.2`（2025-07-07 发布，V1 时代），当前版本 2.9.0；现用的 `Sandbox(domain=..., api_key=...)` 构造式在 v2 已改为 `Sandbox.create(...)`。 + +**E2B 在什么场景下仍然是对的**：① bisheng 若出 SaaS/公有云版本，且用户与数据都在境外——E2B 的 Firecracker microVM 是真 VM 级隔离，pause/resume 保存内存态（resume ≈1s，暂停态无 TTL），冷启动优势明显；② 开发调试期需要一个「不脏本机、可抛弃」的执行环境；③ 客户本身已在 AWS/GCP 上、且愿意接受 Cloudflare 依赖的托管型部署。**利好一条**：`e2b_executor.py:88` 已支持传 `domain` 参数，正是自托管集群的接入方式，代码层对接自托管无需改造——只是那个集群本身建不起来。 + +--- + +## 5. deepagents 官方到底推荐了什么 + +**不存在官方推荐。** 对 docs.langchain.com 全站做 `recommend*` 与 E2B/Daytona/Modal/Runloop/Vercel/AgentCore/OpenShell 同句共现的正则检索，**命中数为 0**。[sandboxes 文档](https://docs.langchain.com/oss/python/deepagents/sandboxes)全页唯一一处 "We recommend" 指向 LangSmith Engine（可观测性产品），与沙箱选型无关；"Available providers" 一节全文只有两句：指向集成清单，以及 "Don't see your provider? You can implement your own sandbox backend." + +官方**唯一带推荐语气的是两条原则性结论**：`"a sandboxed environment is highly recommended for any production use"`，以及 `"FilesystemBackend and LocalShellBackend access the host directly. Don't use them in deployed agents."`——后者正好定性了 bisheng 现在的 local 模式：官方把 `LocalShellBackend` 的 inappropriate use cases 明列为「Production environments (web servers, APIs, multi-tenant systems)」与「Processing untrusted user input」，bisheng 两条全中；并注明 `virtual_mode=True` 在开了 shell 之后 "provides no security"。 + +**但也不能反过来说「所有 provider 一视同仁」**——代码层存在清晰的四级梯队（这是真实的结构性偏好，只是官方从不用「推荐」二字）： + +| 梯队 | provider | 证据 | +|---|---|---| +| ① 第一方硬依赖 | **LangSmith Sandbox** | 唯一打包进库本体（`backends/langsmith.py`），是 `deepagents-code` 的基础依赖 `langsmith[sandbox]>=0.10.10`，文档所有示例的第一个 Tab，CLI 文档原文 "LangSmith is included; other built-ins require extras"，并在 0.7.x 独享 `enable_capture_offload = True` | +| ② monorepo 内 extra | Daytona / Modal / Runloop / Vercel | 源码就在 `langchain-ai/deepagents` 的 `libs/partners/` 里，计入 `all-sandboxes` | +| ③ 外部包 extra | AgentCore | 是 extra、计入 `all-sandboxes`，但包不在 monorepo 内 | +| ④ 明确的第三方 | **E2B** | `libs/partners/` 无 e2b；`libs/code/pyproject.toml` 的 sandbox extras 五项无 e2b；官方原文 "E2B is a **third-party provider** published by the `langchain-e2b` package. Install it as a package, not a `deepagents-code` extra"，且 "The `all-sandboxes` extra **does not include** third-party providers such as E2B"；`langchain-e2b` 由 E2B 自己维护（[github.com/e2b-dev/langchain-e2b](https://github.com/e2b-dev/langchain-e2b)） | + +**结论：bisheng 用 E2B 不是「跟着官方走」，而是选了官方集成清单里梯队最低的一项。** + +**官方集成清单及私有化适配性**（按月下载量排序，取自 [integrations/sandboxes](https://docs.langchain.com/oss/python/integrations/sandboxes)）： + +| provider | 月下载 | 私有化适配性 | +|---|---:|---| +| Daytona | 240k | ❌ 开源仓 2026-06 起停维护、核心闭源、AGPL-3.0 | +| Modal | 88k | ❌ 纯 SaaS，官方无 BYOC。唯一可借鉴：gVisor 隔离 + `blockNetwork: true` 断网 | +| AgentCore | 62k | ❌ 需 AWS 账号 | +| Runloop | 25k | ❌ 仅 Enterprise 档给 VPC 部署，且 VPC ≠ 客户内网 | +| Vercel | 24k | ❌ 纯 SaaS，且已要求 deepagents 0.7.x | +| **E2B** | 17k | ❌ 见 §4 | +| NVIDIA OpenShell | 334 | ⭕ **官方名单里唯一能全内网自建**（Apache-2.0，Landlock LSM + seccomp BPF + OPA/Rego，compute driver 可选 Docker/Podman/K8s/MicroVM），但自承 alpha "Expect breakage"，且 Landlock 需内核 5.13+，对信创常见 4.19/5.10 内核是硬风险 | +| LangSmith | N/A | ❌ 自建强制 K8s（仅 EKS/GKE，连 AKS 都不支持）+ `/dev/kvm` + JuiceFS + Redis + S3 | +| `langchain-sandbox`（LangChain 自研 Pyodide+Deno） | – | ❌ **已归档**，README 原文 "no longer maintained... We do not recommend using it for any production use cases"，且 "File access: Currently not supported" | + +**两条对 bisheng 直接可用的官方结论**： + +- **接沙箱的门槛极低**：`"the only method a provider must implement is execute()... Adding a new provider is straightforward."` 其余文件操作由 `BaseSandbox` 用 shell 脚本拼出来。bisheng 完全可以自己写 `BishengSandbox(BaseSandbox)` 打到任何自建隔离层。 +- **技能包不可见有官方标准解法**：`skills.mdx` 明确 "Skill files stored outside the sandbox are not available inside it"，给的模式是 `before_agent` 中间件 `upload_files` 进沙箱、`after_agent` `download_files` 回写，并附完整的 `SkillSandboxSyncMiddleware` 示例代码。 + +**关于 deepagents 版本，需要修正前序调研的一个错误**：前序称「0.6.12 与 0.7.5 的 backends 目录文件完全一致」——**这是错的，它只比对了文件名**。逐文件 blob SHA 比对显示 11 个文件全部不同，`sandbox.py` 从 39,484 增长到 65,204 字节（+65%）。准确表述是：**provider 对接契约稳定**——`SandboxBackendProtocol` 类在 0.6.12 与 main 之间逐字节一致，Python 侧也没出现 JS 那种 `BackendProtocolV2`，所以「自研 backend 不必升级 deepagents」这条判断成立；但沙箱周边在 0.7.0 有实质增量，其中两条与 bisheng 相关：① 0.7.0 修了一个**沙箱路径穿越**问题（#4588：约束 glob/grep 到 search root、拒绝 `..`、过滤软链外溢），停在 0.6.12 意味着不带这个加固；② `langchain-e2b` 最新 0.0.6 仍 pin `deepagents>=0.6.0,<0.7.0`，未来升 0.7.x 时官方 E2B 适配包会成为阻塞项。 + +**顺带纠正一个常见误解**：`Interpreters`（QuickJS 跑 JS 的 `eval` 工具）是 deepagents 的第二条代码执行路线，但它无 shell、无装包、无文件系统和网络，**替代不了** bisheng 的 code interpreter。 + +--- + +## 6. 推荐路线图 + +### 短期（现在 ～ 1 个月）：止血，与沙箱选型无关 + +**做什么** + +*114 / 测试环境补依赖*（1～2 人天）： +- `dnf install libreoffice-impress`（114 只装了 writer，pptx→pdf 必失败） +- 修 npm（node v22.17.1 在但 `npm install` 静默失败） +- `pip install defusedxml` —— **这是比 node/markitdown 更硬的阻塞点**：`uv.lock` 中查无该包，会让 pptx 的 `clean.py`/`thumbnail.py`/`office/validate.py` 与 docx 的 `merge_runs.py`/`comment.py`/`validate.py` **在第一行 import 就 ImportError** +- 实测并补齐 `poppler-utils`(pdftoppm) 与 `zip`/`unzip`：三个 skill 的视觉 QA 与 pptx/docx 的「unzip → 改 XML → zip」编辑主干都依赖。base.Dockerfile 的 apt 列表里没有它们，`python:3.11-slim` 默认也不含——**镜像内跑一次 `which pdftoppm zip unzip node npm markitdown && python -c "import defusedxml"` 做一次性确权**，成本极低 +- 把 `local_executor.py` 硬注入的 `family="WenQuanYi Zen Hei"` 改成按可用字体探测（114 是 Noto CJK，官方镜像是 WenQuanYi，E2B 模板也是 Noto CJK——三处不一致） + +*local 模式最小加固*（3～5 人天，**不换架构**）： +- **Fernet 密钥外置**：`settings.py:20` 的 `secret_key = "TI31VYJ-..."` 是开源常量，意味着所有客户共用一把密钥，config.yaml 里 `encrypt(...)` 的 DB/Redis 口令对任何在后端进程里跑的代码等于明文。这是独立于沙箱选型的高危问题，应单独立项 +- **`subprocess.Popen` 传 `env=` 白名单**：当前不传 env，子进程全量继承 MinIO AK/SK、网关 HMAC 密钥 +- **加 `preexec_fn` setrlimit**（内存/进程数/文件数）+ compose 加 `mem_limit`/`cpus`/`pids_limit`：当前唯一防线是 600s 超时，一个 fork bomb 可以把后端和 Celery worker 一起拖垮。参照同行判例：FastGPT 就因「只有应用层软限、Docker 未设 cgroup 硬限」被挂了 [GHSA-qv7v-r94x-6x3x](https://github.com/labring/FastGPT/security/advisories/GHSA-qv7v-r94x-6x3x)（CWE-400/770），而 bisheng 连那个 512MB 软限都没有 +- **去掉 compose 的 `security_opt: seccomp:unconfined`**：Docker 默认 seccomp 挡约 44 个 syscall，零维护成本 +- **修 `_walk_sandbox` 的 depth 问题**（`files.list('./')` 默认非递归导致 `new_files` 契约失效）与**失败路径丢弃 stdout** 的问题 + +**不做什么**：不动沙箱架构、不引入新组件、不为此升级 deepagents、不把 Anthropic 官方 skill 原样打包分发（LICENSE 专有条款禁止 reproduce/derivative works，只能参照重写）。 + +**判据**：114 上 pptx/docx/xlsx 三条 skill 主干能各跑通一次端到端；`grep -rn "seccomp:unconfined" docker/` 无输出；执行器进程的 `/proc//environ` 不含 MinIO 密钥。 + +**顺带修正对官方 skill 依赖的判断**（前序调研把阻塞点指错了）：node 依赖只覆盖 **pptx 与 docx 的「创建」分支**（SKILL.md 写死 "`pptxgenjs`/`docx` is preinstalled — do not run `npm install` first"）；**xlsx 的 SKILL.md 全文零 node、零 npm、零 JS**，主干就是 openpyxl + pandas + soffice，在 bisheng 官方镜像里**零缺口可直接跑通**。markitdown 只在 pptx 里是实质阻塞（"Content QA (required)" 的唯一文本读取路径），在 xlsx 里只是次选（SKILL.md 自注 "No cell coordinates, so don't plan edits from it"），**docx 通篇不提 markitdown**，读取用 `pandoc -t markdown`——而 bisheng base 镜像手工装了 pandoc 3.6.4，这条反而是通的。所以准确分档是：**xlsx 主干可跑通 / docx 只死「创建」一半 / 只有 pptx 是真·大面积失效**（创建=node、读取=markitdown、清理校验=defusedxml、视觉 QA=pdftoppm，四条全断）。另注意 114 与官方镜像的失败面**不相同**：114 有 node 但缺 impress，官方镜像有 impress 但缺 node，不能混为一谈。 + +--- + +### 中期（1～2 个季度）：落地「独立容器 + compose 原生加固」 + +**做什么**：新增第三种执行模式 `container`，实现为独立容器 + docker-py 驱动。两条实现路径二选一： +- **A. 自建 `bisheng-code-runner`**（总分 87.6）——完全可控，与现有 `BaseExecutor` 契合最好 +- **B. 直接用 [llm-sandbox](https://github.com/vndee/llm-sandbox)**（总分 85）——省掉容器生命周期与文件搬运的自研工作量。经源码级核实：`llm_sandbox/docker.py` 内 `put_archive`/`get_archive` 实现了双向 `copy_to_runtime`/`copy_from_runtime`，支持 `image=` 与 `dockerfile=`，`config.py` 的 `skip_environment_setup` 配合预拉镜像用于[官方文档明示的 air-gapped 部署](https://vndee.github.io/llm-sandbox/configuration/)。风险是**巴士系数 1**（贡献者 20 人但 vndee 一人 545 次提交、次位是 dependabot 27 次） + +**实施要点**： + +1. **`BaseExecutor` 需要新增什么**——实测契约极薄：`__init__(minio: dict, **kwargs)` + `@abstractmethod run(code) -> Any` + 可选 `close()`，另加隐式的 `description` property（`load_tools.py:112` 用到但基类没声明）。但**真正要满足的隐式契约更大**：`run` 返回值形状两家完全不同（local 返 `{exitcode, log, file_list}`，E2B 返 `{results, stdout, stderr, error, file_list, new_files}`，这份 dict 原样成为 ToolMessage 内容，**没有归一层**），签名也不一致（E2B 多一个 `required_files`），产物回收靠 `local_sync_path` 这个约定俗成的 kwarg。**建议本轮顺手把返回值归一化**，否则第三种模式会带来第三种形状。 +2. **镜像**：从现有 `base.Dockerfile` 派生 `bisheng-sandbox`（已含 libreoffice 元包 + pandoc 3.6.4 + fonts-wqy-zenhei + ffmpeg + python-pptx/docx/openpyxl/pymupdf），补 **node/npm + defusedxml + poppler-utils + zip/unzip**，统一中文字体。**「沙箱镜像 = 唯一可信环境」是走沙箱的第二个理由**（第一个是安全）——114 那批「手动部署漂移」问题会被一次性收口。 +3. **网络隔离直接抄 Dify**，纯 compose 内可做、宿主机零改动：执行器只接在独立 network 上，无路由到 mysql/redis/minio/openfga；出网走 Squid 白名单或直接 `network:none`。Dify 的 `local_sandbox` 注释原文即 "has NO direct route to `api`"。当前 bisheng 的官方 compose 是 mysql `root/1234`、minio `minioadmin/minioadmin`、ES 明文 http、OpenFGA 8080 全部可达——**读其它租户数据不需要绕过任何权限层，直接连库即可**。 +4. **配置**：`gpts_tools.extra` 的 `type` 字段加第三个值 `container`，`config.container` 段承载镜像名/资源限额/网络策略；前端 `CodeExecutor.tsx:124-141` 的 RadioGroup 加一项。顺便修一个现存小 bug：前端提交体把 `config.e2b.type`（private/official）塞进 e2b 段，后端 `kwargs.update(config)` 会把它当 `type` kwarg 传给 `E2bCodeExecutor` 被 `**kwargs` 吞掉——真正区分自建/官方的只有 domain 传不传。 +5. **技能包挂载时序是一等公民**（这是换沙箱的最大红利）：copy-in 时机由 bisheng 自己控制，可在容器启动后再灌技能包，从根上解掉「快照早于技能物化」。三个方案需明确选一：每任务 `put_archive` 灌入 / 烘进镜像 / 挂只读 volume——在多租户 + 技能版本更新下语义差异很大。 +6. **配置面必须不可写**（这条容易漏）：[Cymulate 2026-05 的研究](https://cymulate.com/blog/the-race-to-ship-ai-tools-left-security-behind-part-1-sandbox-escape/)证明 Claude Code / Gemini CLI / Codex CLI 三家的逃逸根因都是「沙箱被当成边界，而真正的边界——宿主侧配置与执行逻辑——从沙箱内可写」。bisheng 现有 compose 把 `config.yaml` 与 `entrypoint.sh` 以**可写**方式 bind-mount 进容器，正好命中这个模式，必须对执行器不可见或至少 `:ro`。 + +**已验证的加固基线（2026-08-11 POC，可直接抄进 compose）**： + +```yaml +# 八档逐级加固实测全绿，这一组就是可落地的目标态 +cap_drop: [ALL] +security_opt: [no-new-privileges:true] # 保留 Docker 默认 seccomp，不要 unconfined +user: "65534:65534" # 非 root +read_only: true # 根文件系统只读 +tmpfs: + - /tmp:rw,size=512m # Docker 默认已含 noexec,nosuid,nodev +networks: [sandbox_only] # 无路由到 mysql/redis/minio/openfga +mem_limit: 1g # 见下方内存下限实测 +pids_limit: 256 +cpus: 2 +environment: # 只读 rootfs 下这三个必须显式指向 tmpfs + HOME: /tmp/home + XDG_CACHE_HOME: /tmp/cache + TMPDIR: /tmp +``` + +**内存下限实测**：单次转换在 **128Mi** 下仍能完成；**4 路并发的下限是 384Mi**，256Mi 及以下并发档必被 OOM killer 杀（`rc=137`，且 soffice 只吐一句无关的 javaldx 警告，**报错信息完全指不到内存**——排障时容易误判）。建议 `mem_limit` 取 1g 留余量。 + +**人天量级估算（路径 A 自建）**： + +| 工作项 | 人天 | +|---|---:| +| `ContainerExecutor(BaseExecutor)` + 返回值归一化 | 5～8 | +| 沙箱镜像（派生 + 补依赖 + 字体统一 + 体积优化） | 2～3 | +| compose/网络/cgroup/只读挂载 + 前端表单 + extra schema | 3～4 | +| 凭据收口（Fernet 外置、env 白名单）※ 与短期项合并可省 | 2～3 | +| 技能包/工作区挂载时序 + 产物收割对齐 | 3～5 | +| E2E 回归（soffice / pandoc / node / playwright 四条链路 × 三个 skill） | 4～6 | +| **合计** | **约 20～30 人天** | + +路径 B（llm-sandbox）可省约 5～8 人天，但增加一个巴士系数 1 的外部依赖。 + +**不做什么**：不把 K8s 当作**前提**——执行器的容器生命周期要走一层薄抽象，compose 下用 docker-py、客户已有 K8s 时能换成 Job/Pod，**两种编排都能跑**，但绝不因此要求客户新上 K8s（LangSmith Sandbox 仍出局：它强制 EKS/GKE + `/dev/kvm`，是「强制上 K8s」而非「支持 K8s」）；不自研 seccomp 系统调用白名单（[dify-sandbox 的 FAQ](https://github.com/langgenius/dify-sandbox/blob/main/FAQ.md) 整节在教用户怎么用 strace 挖 syscall 号补白名单，社区还有 0.2.15 白名单不全导致间歇性失败的 issue——用 Docker 默认 profile 即可拿到大部分收益）；不抄 RAGFlow 的控制面设计（它的 `executor-manager` 是 `privileged: true` + 挂 `/var/run/docker.sock`，等于把宿主 root 交给一个对外提供代码执行的网络服务，等保测评几乎必判高风险）。 + +**判据**：`container` 模式下三个 office skill 全链路跑通；执行器容器内 `curl mysql:3306` 不通；`docker inspect` 显示 `mem_limit`/`pids_limit` 生效、`CapDrop=ALL`、`ReadonlyRootfs=true`；技能包在容器内可见且可执行；产物清单（`new_files`）能列出 `output/` 子目录文件。 + +**对客户与安全评审的口径要诚实**：共享宿主内核，NIST SP 800-190 明说容器不是 VM 级安全边界，定位是「挡住误操作与机会主义攻击」，**不承诺抗定向内核 0day**。同行判例佐证单层不够：Dify 做到了「seccomp 白名单 + chroot + 非 root」仍被 [CVE-2024-10252](https://www.varmor.org/blog/harden-the-AI-application-development-platform/) 经 preload 字段逃逸、拿到容器 root。 + +--- + +### 中期可选加固档：叠加 gVisor + +**在客户对「不可信代码硬隔离」有硬要求时启用。** gVisor 是**唯一**在「VMware 虚机 + 无嵌套虚拟化」条件下仍能提供强隔离的方案：默认 systrap 平台基于 seccomp `SECCOMP_RET_TRAP`，[官方文档](https://gvisor.dev/docs/architecture_guide/platforms/)原文 "works in virtualized environments without requiring nested virtualization support"，且 systrap 自 2023 年中取代 ptrace 成为默认平台，在嵌套 VM 场景下通常比 KVM 平台**更快**。 + +**最强旁证**：Freedom of the Press Foundation 的 [Dangerzone](https://gvisor.dev/blog/2024/09/23/safe-ride-into-the-dangerzone/)（5.7k★，活跃）自 v0.7.0 起就是「在一个**非 privileged** 的 Docker 容器里再套一层 gVisor 跑 LibreOffice 做文档转换」，外层容器只需 `CAP_SYS_CHROOT` + seccomp 放行 `ptrace`，**容器镜像无需改造**，官方博客确认 "LibreOffice operates successfully within the nested setup"。这几乎是 bisheng 场景的 1:1 复刻。 + +**代价**：越过「compose 一把梭」一格（root 装 runsc + 改 daemon 配置 + 离线备对应架构 tarball）；**龙芯无解**（官方只支持 x86_64/ARM64）；麒麟 V10 的 4.19.90 内核名义满足 4.14.77+ 门槛但**无任何官方适配或信创兼容性认证**；gVisor 官方明说 "resource limits are not enforced within the sandbox"，配额仍要靠宿主 cgroup 兜底。**+3～5 人天**（含实测）。 + +--- + +### 长期：多后端可插拔抽象 + +目标形态：`local / e2b / container(+runsc) / <未来>` 四选一，且新增后端不改调用方。设计建议： + +1. **把 `run` 返回值升为显式契约**（TypedDict 或 dataclass），字段至少 `{exitcode, stdout, stderr, error, artifacts: list[Artifact]}`，`Artifact = {rel_path, size, zone, is_product}`。当前三种模式吐三种 dict 且原样进 ToolMessage 的做法，是「换后端必炸提示词」的根源。 +2. **`description` 提为抽象属性**，并把「环境有什么、能不能 pip install」写进各自实现——现在 local 明写「offline environment, do NOT pip install」，E2B 完全没有库清单也没提 pip，**两个模式给模型的环境认知是断层的**。 +3. **把 copy-in / copy-out 抽成显式生命周期钩子**（`prepare_workspace(paths)` / `collect_artifacts() -> list[Artifact]`），对齐 deepagents 官方的 `before_agent`/`after_agent` 中间件模式。技能包挂载时机由此变成后端无关的一等公民，而不是每个后端各自踩坑。 +4. **保留 `local` 但改为「开发/单机默认关闭」**，参照 [Open WebUI 的硬化文档口径](https://docs.openwebui.com/getting-started/advanced-topics/hardening/)：不需要就关掉；需要就隔离到独立容器；不要把浏览器端 Pyodide 当安全边界。 +5. **不要为了「与官方对齐」去实现 `deepagents.BaseSandbox`**——官方门槛虽低（只需 `execute()`），但 bisheng 的 `BaseExecutor` 已经是自己的抽象，且 0.6.12 契约稳定、无升级压力。若将来确要对齐，走 `SandboxBackendProtocol` 即可，Python 侧没有 V2 协议迁移风险。 + +--- + +## 7. 风险与未决问题 + +**必须 POC 验证的假设（阻塞中期方案）** + +| # | 待验证 | 状态 | +|---|---|---| +| 1 | 受限容器下 `soffice --headless --convert-to pdf` 能否跑通 | ✅ **已验证通过（2026-08-11）**，见下方「POC 记录」。八档逐级加固全绿，命门解除 | +| 2 | 同上条件下 pandoc / PyMuPDF / node / playwright+chromium | ⚠️ **部分验证**：pandoc 3.6.4、PyMuPDF 1.26.5 已在最严档跑通；**node 与 playwright+chromium 未测**（两者都不在 office 链路上；chromium 需要 user namespace，在 `--cap-drop=ALL` 下大概率要 `--no-sandbox`，是**独立的待验项**，不要因为本轮全绿就默认它也没问题） | +| 3 | 若叠加 gVisor：上述在 runsc 下是否跑通 | ❌ **未验证**。Docker Desktop 不带 `runsc`，本轮测不了。gVisor 官方立场仍是 "the only real way to know if it will work is to try"；Dangerzone 只证明了 doc→PDF，未覆盖 impress 路径。**这是可选加固档的前置，不阻塞主推方案** | +| 4 | 官方镜像内 `which pdftoppm zip unzip` 与 `import defusedxml` | ✅ **已验证：全部缺失**（`pdftoppm` / `pdftotext` / `zip` / `unzip` / `node` / `npm` / `markitdown` / `gs` / `convert` 均 MISSING；`defusedxml` MISSING，确认不是 python-pptx/docx/openpyxl/lxml 的传递依赖）。同时确认 **`libreoffice` 元包确实带 `scalc` / `simpress` / `swriter` / `sdraw`**（LibreOffice 25.2.3.2），§2 关于「官方镜像有 Impress/Calc」的判断成立。残余不确定：真实镜像还跑了 `playwright install-deps`，理论上可能额外带入个别包，但它装的是字体与图形库，不含 poppler-utils / zip | +| 5 | llm-sandbox 能否透传 `runtime='runsc'` | `docker.py:378` 有 `container_config.update(self.config.runtime_configs)`，理论可行但**官方文档零记载、未验证**，纯源码推断 | +| 6 | llm-sandbox 的 `copy_to_runtime` 能否在 session 启动**之后**调用 | 若只能在创建时传，「技能包不可见」会原样复现，是选型 A/B 的分水岭 | + +**环境事实未查实** + +- **宿主机形态：已部分收口（2026-08-11 产品侧口径，未逐客户核实）**——「大概率是虚机，基本不会是公有云/专有云云主机」。这带来两条确定性：① **公有云裸金属 SKU 那条路可以从选型里删掉**，采购成本一档不再是考虑项；② **只要主流是虚机，microVM 路线就不能作为默认方案**——它在虚机里必须由客户虚拟化团队逐台开启嵌套虚拟化，属于「要客户改配置」的实施阻力，即使技术可行也不该进默认交付形态。**仍待确认的只剩虚拟化平台厂商**：VMware（Broadcom KB 313547 明确不支持在虚机内跑第三方 hypervisor ⇒ microVM 全档划掉）vs KVM 系（FusionCompute / ZStack / H3C CAS / OpenStack / 深信服 ⇒ 可开，但仍是逐台手动开关）。取证方式：客户机器上 `systemd-detect-virt`（none=物理机）+ `ls -l /dev/kvm`。 +- **物理机占比未知**——用户表述为「有概率是虚机」，非「全部是虚机」。物理机那部分客户可作为 microVM 可选升级档的候选，但不改变默认方案。 +- **阿里云普通 ECS 是否支持嵌套虚拟化存疑**——搜索索引存在《如何为 ECS 实例开启嵌套虚拟化》页面且摘要含具体细节（控制台「更改 CPU 选项」、支持 g7/c7/r7 等），但实测该 URL 301 重定向到裸金属规格页。**这是唯一可能推翻「普通云虚机不支持」的点**，建议用控制台或 ECS OpenAPI 实证。 +- **华为云对嵌套虚拟化无任何官方口径**（逐页检查 BMS 产品优势/什么是 BMS/产品功能，0 次提及）。**报告中不应对华为云下断言**。 +- **VMware「默认关闭」无字面原文**——Broadcom 文档只给「主动勾选」的操作步骤，结论由步骤 + vSphere API `nestedHVEnabled` 语义（未设置时不自动开启，除非 `vbsEnabled=true`）推导。若需强硬依据，改引 KB 313547 的「官方不支持嵌套 hypervisor」，那条是字面原文且杀伤力更强。 +- **国产化 CPU 全场无实证**：gVisor / Kata / Firecracker / microsandbox / Sysbox **没有任何一家**给出针对鲲鹏 920、飞腾 D2000/S2500、海光 C86、兆芯开先的官方验证声明。龙芯 loongarch64 全组无解，若有龙芯客户需单独立项。 +- **需要注意一个反向事实**：[Linux 内核官方文档](https://docs.kernel.org/virt/kvm/x86/running-nested-guests.html)明确 "From Linux kernel v4.20 onwards, the `nested` KVM parameter is enabled by default for Intel and AMD"，国内私有云/信创底座（FusionCompute、ZStack、H3C CAS、深信服、OpenStack）多为 KVM 系，**宿主侧默认是开的**；阿里云弹性裸金属文档原文更是「二次虚拟化能力**默认开启**」。所以对外表述必须分层：**VMware 客户处基本不可行；物理机与 KVM 系私有云上是「可开、需管理员配合」；公有云需换裸金属 SKU（采购成本上一档）**——不能笼统说「嵌套虚拟化默认关闭」，会在客户/评审面前翻车。 + +**合规与法务未闭合** + +- **等保 2.0 三级测评对「平台内置代码执行器」的具体判定口径**（属恶意代码防范？入侵防范？还是应用安全）未取到条款级证据。GB/T 22239-2019 原文未直接抓取，建议由合规同事补一条权威引用——这直接影响「加固」要做到哪一档才算过线。 +- **Judge0 是否另有商业授权/例外条款**未查到（judge0.com 有商业页面与 Judge0 Extra CE）。若将来考虑，需先确认。 +- **Anthropic skill 的 LICENSE 专有条款**已核实（禁止 extract/reproduce/derivative works），但「参照重写到什么程度算 derivative work」需法务判。 + +**其它已知盲区** + +- **Anthropic 托管环境的完整依赖清单从未公开文档化**。公开的 [code execution container 规格](https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool)只有 Python 3.11 + pandas 那批 + `unzip/unrar/7zip/bc/rg/fd/sqlite`，**没有 node、没有 LibreOffice、没有 pandoc、没有 markitdown**，且完全断网。这段「未文档化落差」无法用一手来源闭合，只能靠实测反推——这正是「照搬官方 skill 必踩坑」的结构性根因。 +- **官方 skill 对非 Claude 模型不可迁移的三条假设**：① 视觉 QA 要求「inspect every one」的 slide-*.jpg（纯文本模型做不到，Qwen-VL 需灵思把图片回灌进上下文，而 bisheng 代码执行器当前不做这件事）；② xlsx 要求恰好六个函数写 `_xlfn.` 前缀并禁用 XLOOKUP/FILTER/UNIQUE 等，SKILL.md 亲口承认此时 `recalc.py` 会报 `total_errors: 0`——**校验器抓不住，纯靠模型自律**；③ 大量裸 shell 指令（`markitdown … | grep`、`(cd unpacked && zip -Xr ../out.pptx .)`）与「do not run npm/pip install」的硬指令组合，会让离线环境下的模型既不装依赖也不换路径，直接空转。 +- **OpenSandbox 增速异常**（8 个月 12.4k★），建议复核 contributor 分布与非阿里体系生产用例后再定权重；其离线部署与 arm64/国产 CPU 支持均无官方文档。 +- **microsandbox 有 Daytona 式的商业化风险**：Super Rad Company 云服务闭测中，形态与 Daytona 转闭源前高度相似；其 libkrunfw guest kernel 作为 release asset 分发，若上游停发则离线部署不可持续。 +- **GPU 路径完全未评估**。gVisor 支持 NVIDIA GPU（nvproxy）与 TPU，但各方案差异很大；有二手来源称 gVisor 层会阻断 GPU 直通，未经一手证实。若灵思后续要跑本地推理或 GPU 加速数据分析，需单独立项。 + +--- + +## 8. POC 记录：受限容器下的 office 链路（2026-08-11 实测） + +**结论：整条 office 产出链路在最严加固档下完全可用，八档全绿 12/12。**§7 第 1 项（原判为「整条 office 链路的命门」）解除。 + +**被测镜像**：`python:3.11-slim` + 与 `base.Dockerfile` 逐字相同的 apt 行（`--no-install-recommends libreoffice wget procps fonts-wqy-zenhei libglib2.0-0 libsm6 libxrender1 libxext6 libgl1`）+ 同版本 pandoc 3.6.4 + python-pptx 1.0.2 / python-docx 1.2.0 / openpyxl 3.1.5 / pymupdf 1.26.5。剥掉了与文档转换无关的部分（全量 uv sync、playwright、ffmpeg）。LibreOffice 实测版本 25.2.3.2。 + +**每档执行的 12 项**：python-docx/openpyxl/python-pptx 各生成一个含中文的文件 → soffice 分别把 docx/xlsx/pptx 转 PDF → pandoc 读 docx → PyMuPDF 渲染成 PNG → 4 路并发转换 → CJK 字体枚举 → 网络连通性反向校验。 + +| 档 | 相对上一档新增的约束 | 结果 | +|---|---|---| +| T0 | 基线 | 12/12 | +| T1 | `--cap-drop=ALL` `--security-opt=no-new-privileges` | 12/12 | +| T2 | `--network=none` | 12/12 | +| T3 | `--user 65534:65534`（非 root） | 12/12 | +| T4 | `--read-only` + `--tmpfs /tmp:rw,size=512m` | 12/12 | +| T5 | `--memory=1g --pids-limit=256 --cpus=2` | 12/12 | +| T6 | `/tmp` 显式 `noexec,nosuid,nodev` | 12/12 | +| T7 | 内存压到 512m、pids 压到 128 | 12/12 | + +最严档（T7）容器内自检：`uid=65534 user=nobody rootfs=ro CapEff=0000000000000000 tmp=[rw,nosuid,nodev,noexec,relatime,size=524288k] mem=512Mi`，网络校验返回 `blocked (OSError)`。三次转换各 0.7–0.9s，4 路并发 2.9s。 + +**三个值得记的实测发现** + +1. **Docker 的 `--tmpfs` 默认就带 `noexec,nosuid,nodev`** —— 原本预判最可能断的 T6，其实从 T4 起就一直生效。LibreOffice 在 noexec 的 `/tmp` 上正常工作。 +2. **只读 rootfs 下必须显式把 `HOME` / `XDG_CACHE_HOME` / `TMPDIR` 指向 tmpfs**，否则 fontconfig 缓存与 LibreOffice profile 无处可写。这是唯一需要为加固额外做的适配。 +3. **中文渲染正常，非豆腐块**（导出 PNG 目视确认，非仅凭 PDF 可抽取 CJK 文本 —— 文本层有 CJK 不等于字形渲染正确，两者必须分开验）。 + +**口径与残余风险（不可略过）** + +- 宿主是 **Docker Desktop 的 linuxkit 内核 6.12.69 / x86_64**，**不是**客户内核（信创常见 4.19/5.10）。capability、seccomp、read-only、cgroup 均为标准语义，结论可移植，但**上线前必须在目标内核上重跑同一套脚本**。 +- **未覆盖**：gVisor（`runsc` 本机不可得）、node、playwright+chromium。chromium 在 `--cap-drop=ALL` 下大概率需要 `--no-sandbox`，属独立待验项。 +- 本轮验的是**能不能跑通**，不是**逃不逃得出去**。容器仍共享宿主内核，对外口径按 §6 末尾那段说。 + +**复现**：探针与矩阵脚本在 `scratchpad/poc/{Dockerfile,probe.py,run_matrix.sh}`（会话临时目录，未入库；如需长期保留应移入 `src/backend/scripts/`）。 diff --git a/docs/linsight-skill-authoring.md b/docs/linsight-skill-authoring.md new file mode 100644 index 0000000000..08922b5e6b --- /dev/null +++ b/docs/linsight-skill-authoring.md @@ -0,0 +1,208 @@ +# 灵思技能包编写指南 + +本文讲**灵思（Linsight）任务模式**技能包怎么写、怎么随部署分发。 +与仓库里的 `.claude/skills`、`.agents/skills`（Claude Code 自己的技能）**没有任何关系**，两者互不通用。 + +## 内置技能：放哪、怎么生效 + +源码放在 **`src/backend/bisheng/linsight/builtin_skills//`** —— 必须在 `src/backend/` 之内， +因为 `src/backend/Dockerfile` 是 `COPY ./ ./`，构建上下文就是 `src/backend`，仓库根的目录不会进镜像。 +放在包内之后，docker 镜像、裸机 rsync、pip 安装都自动带上。 + +| 目录 | 说明 | +|---|---| +| `bisheng-pptx/` | PPT 制作技能（BiSheng 适配版）。用 python-pptx 生成 .pptx，含中文排版规范、模板套用、交付前自检脚本。用户要「PPT／幻灯片／汇报材料」时用 | +| `bisheng-xlsx/` | Excel 表格技能。用 openpyxl 生成 .xlsx，交付前必须经 LibreOffice Calc 重算公式再体检。用户要「表格／报表／台账／预算测算／财务模型」时用 | +| `bisheng-docx/` | Word 文档技能。用 python-docx 生成 .docx，覆盖 `w:eastAsia` 中文字体、目录域、页码、表格列宽、封面。用户要**带版式**的正式文档时用；只是「把刚才的内容存成 Word」走自带的 `export_docx`，别用本技能 | + +生效链路是**应用启动时自动 seed**（`domain/services/builtin_skill_seeder.py`，挂在 `main.py` 的 +lifespan 上，和既有的两个 backfill 同一位置）：读包 → 遍历活跃租户 → 写入 +`SKILLS_ROOT/data/skills/{tenant_id}//` 并建 `linsight_skill` 行（`source='builtin'`）。 +所以 `docker compose up` 起来技能就在选择器里，**不需要任何运维脚本**。 + +几条设计约束，加新内置技能前先了解： + +- **幂等靠内容比对**：磁盘上已装的 bundle 与镜像里的逐字节比，不同才重写。升级镜像重启即更新， + 没变的话只花几次文件读取。`__pycache__`/`.pyc`/`.DS_Store` 被 seeder 与打包脚本一致跳过， + 所以本地跑过脚本留下的 `__pycache__` 不会让每次启动都判成「变了」。 +- **用户改过的永不覆盖**：管理端编辑内置技能会把 `source` 翻成 `manual`（`SkillService._mark_forked`）， + 该租户的副本从此不再被 seed 覆盖。这是刻意的——升级时静默回滚客户的修改，比让副本漂移糟糕得多。 +- **新租户会补种**：启动 seed 只覆盖当时存在的租户，所以 `TenantService.acreate_tenant` 的 Step 6 + 也会为新租户 seed 一次。 +- **每租户一份物理拷贝**。现在三个包合计约 210KB（pptx 77KB / xlsx 74KB / docx 63KB）， + 100 个租户 21MB 可忽略；但如果将来内置技能带模板库/字体（官方 `presentations` 包 4.7MB）， + 就该改走「只读目录 + DB 只存指针」的形态。 +- **不要写进 Alembic 迁移**：项目铁律是 revision 只做 DDL，数据 seed 一律走独立流程。 + +**不打算内置的技能**（客户定制、一次性）不用放这里，直接用下面的打包流程做成 zip 在管理端导入即可。 + +### bisheng-pptx 的环境前提（实测） + +- **依赖已经在镜像里，无需额外安装**。`python-pptx>=1.0.2` 是 `src/backend/pyproject.toml` 的正式依赖， + main / hotfix/2.6.0 / 3.0 各线都有（都在第 81 行）。实测： + 116（`bisheng-backend:release`，2.6）与 180（`v2.6.0-fix2`）容器内 `python-pptx 1.0.2`、 + `PyMuPDF 1.26.6`、`Pillow`、`lxml`、`matplotlib`、`pandas`、`reportlab`、`openpyxl`、`XlsxWriter` 全部在位； + `defusedxml` 和 `markitdown` 都没有（本技能不依赖它们，这正是不照搬官方 skill 的原因之一）。 + > ⚠️ 在容器里核实依赖必须用 **`/app/.venv/bin/python`**，不能用 `python`／`bash -lc "python ..."` —— + > 后者解析到 `/usr/local/bin/python3`（venv 的 base 解释器），看不到 venv 的 site-packages， + > 会把装好的包全部误判成缺失。 +- **需要用户勾选代码执行器**（`bisheng_code_interpreter`），否则无法生成文件。 +- **预览渲染需要 LibreOffice 带 Impress**。镜像（`base.Dockerfile` 装 Debian `libreoffice` 元包） + 实测**有** impress：116、180 上 pptx→pdf 都成功（`impress_pdf_Export` 过滤器）。 + 但 114 这类手动只装了 `libreoffice-writer` 的机器**没有** impress，转换会报 + "source file could not be loaded"。这只影响可选的视觉 QA，不影响 .pptx 的生成与交付， + 脚本会给出明确提示并跳过。 +- **镜像里没有 node/npm、unzip/zip、pdftoppm**（180 实测）—— 官方 skill 的 pptxgenjs 路线、 + 解包重打包路线、pdftoppm 出图路线在这里全部走不通。 +- **`pandoc` 存在，但不用它做 PPT**。发布镜像手工装了 pandoc 3.6.4 到 `/usr/bin` + （`src/backend/base.Dockerfile:21-27`），`pandoc -o out.pptx` 确实能跑，但只产出「标题＋项目符号」 + 的裸版式，配色/版式/图表/图片位置全不可控。SKILL.md 里要写「有，但不用，因为…」—— + 把存在的东西写成「不存在」，模型自己 `which` 到之后会开始怀疑整张环境表。 + +### bisheng-xlsx 的环境前提（实测） + +- **依赖已经在镜像里**。`openpyxl>=3.1.5` 是 `src/backend/pyproject.toml` 的正式依赖（第 56 行）， + `import openpyxl` 直接可用。`XlsxWriter` 也在，但只能新建、不能读改已有文件，也不能和 openpyxl + 混用 —— 一律走 openpyxl。`markitdown` 不在。 +- **需要用户勾选代码执行器**（`bisheng_code_interpreter`）。 +- **公式必须经 LibreOffice Calc 重算**。openpyxl 写出的公式只有公式文本、**没有缓存值**， + 不重算就等于交了一张全空的表（Excel 打开会自己算，但预览、pandas、下游接口读到的全是 None）。 + 包内 `scripts/recalc_check.py` 用 `soffice` 把文件就地重写一遍，再扫结果里有没有 `#NAME?`/`#REF!`。 +- **Calc 组件不是每台机器都有**。发布镜像装的是 Debian `libreoffice` 元包（`base.Dockerfile:12`）， + Calc 在；114 这类手动只装了 `libreoffice-writer` 的机器**没有** Calc，重算会失败。 + 此时脚本如实报错并给降级方案（公式改成 Python 先算好、写数值），**不要假装重算过了**。 +- **不要用「有没有 `scalc` 可执行文件」判断 Calc 在不在**。非 Debian 布局（如 macOS 版 LibreOffice） + 根本没有 `scalc` 这个文件，而 Calc 完全可用 —— 这样探针会 100% 假阴性，把能用的部署判死。 + 同理，**判重算成败要看文件有没有被重写**（mtime + size），不能只看退出码：soffice 什么都没转也会 exit 0。 + +### bisheng-docx 的环境前提（实测） + +- **依赖已经在镜像里**。`python-docx>=1.2.0` 是 `src/backend/pyproject.toml` 的正式依赖（第 80 行）， + `import docx` 直接可用。 +- **需要用户勾选代码执行器**（`bisheng_code_interpreter`）。 +- **中文字体必须显式写 `w:eastAsia`**。`run.font.name` 只设西文，中文会退回 Calibri/宋体。 + 包内 `set_run_font()` 一次写 ascii/hAnsi/eastAsia 三个属性；只设 latin 的文档看着「设过字体」， + 打开全是错字体。 +- **渲染预览走 LibreOffice Writer，比 pptx 宽松**。发布镜像的 `libreoffice` 元包和 114 那种只装了 + `libreoffice-writer` 的机器**都有** Writer，docx→pdf 两边都转得出来（pptx 需要的 Impress 只有前者有）。 + 没有 `soffice` 时脚本跳过预览，不影响 .docx 的生成与交付。 +- **与自带 `export_docx` 的边界必须在 SKILL.md 里写死**。`export_docx` 走 MarkDocx：只吃 `output/` + 下的 markdown，实测产出 US Letter 21.6×27.9cm、左右边距 3.2cm、表格 `tblW=auto`（无列宽无表头底纹）、 + 无目录域无页码域无页眉页脚、图片固定 5.7 英寸。要 A4、目录、页码、列宽、封面，或要改用户上传的 + .docx，才用本技能。边界不写死，模型会拿重路线做轻活，或者反过来用 `export_docx` 交一份没版式的稿。 + +## 打包与导入 + +```bash +bash scripts/pack_linsight_skill.sh src/backend/bisheng/linsight/builtin_skills/bisheng-pptx +bash scripts/pack_linsight_skill.sh src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx +bash scripts/pack_linsight_skill.sh src/backend/bisheng/linsight/builtin_skills/bisheng-docx +# → dist/bisheng-pptx.zip / dist/bisheng-xlsx.zip / dist/bisheng-docx.zip +``` + +脚本会先校验后端在导入时会卡的那几条（SKILL.md 在包根、frontmatter `name` 等于目录名且是 kebab-case、 +zip ≤10MB、解包 ≤100MB），失败在本地就报出来，不用等管理端返回 11051/11052/11059。 + +导入：管理端 → 灵思 → 技能 → 上传 zip（需要**租户管理员**权限）。技能是**租户私有**的， +Root 租户的技能不会下放给子租户，每个租户都要各导一份。 + +用户侧使用：在任务模式的技能选择器里勾选它。**技能是纯手动勾选，没有自动匹配** —— +没勾选的技能包根本不会被复制进会话工作区，模型物理上看不到。 + +## 写技能包时必须知道的事 + +**格式** + +- `SKILL.md` 必须在包根，带 YAML frontmatter。 +- `name` 必填，`^[a-z0-9]+(-[a-z0-9]+)*$`，≤64，**必须等于目录名**。非法名会被 slugify（中文转拼音）静默改写。 +- `description` 必填，≤1024 字符。**这是唯一进入模型 prompt 的文字** —— 正文完全不注入。 + 技能会不会被用起来，100% 取决于 description 写没写全场景与关键词。 +- `metadata.display-name`：管理页与用户勾选器里显示的名字，可中文，≤255。不写就退回 `name` 那个英文 slug。 +- 其它键（`license`、`allowed-tools` 等）原样保留、不校验。**`allowed-tools` 在 bisheng 里不生效**， + 本轮可用的工具完全由用户在前端勾选的工具决定，技能无法声明依赖。 +- 同租户内 `name` 和 `display-name` 任一重复都会被拒（11055），不会自动加后缀。 + +**运行时** + +- 模型必须自己 `read_file("/skills//SKILL.md", limit=1000)` 才能读到正文（progressive disclosure）。 + 正文里引用别的文件也要给全路径 `/skills//references/xxx.md`。 +- 代码执行器（`bisheng_code_interpreter`）的 cwd 是工作区根，包内文件用**相对路径** `skills//...` 打开。 + 写成 `/skills/...` 指向的是宿主机根目录，`open()` 必 FileNotFoundError + （`base_executor` 的 `_ABSOLUTE_PROVISIONED_RE` 只会附一条提醒，**不拦截**，所以错了也照跑照崩）。 + **两套路径命名空间是刻意区分的，不要「统一」掉**：给 `read_file` 的引用**带**前导斜杠， + 给代码执行器的路径（`skills/`、`output/`、`uploads/`、`scratch/`）**不带**。 +- 调包内脚本一律 `subprocess.run([sys.executable, "skills//scripts/x.py", ...])`。写 `python` 会被 + `infer_lang` 判成 sh，且 PATH 里的 `python` 未必是后端 venv。 +- 提交给执行器的代码里**只要出现三反引号围栏，就只会执行围栏内的内容**（`CODE_BLOCK_PATTERN`）—— + 别在代码参数里贴 markdown。 +- 包里的二进制资产（模板 .pptx、字体、图片）**不能被 `read_file` 读**（会被二进制守卫拦下）， + 只能由代码执行器打开。 +- `output/` 是唯一交付区（每次运行自动创建）；`skills/`、`scratch/`、`uploads/` 都不会被当作交付物。 + `scratch/` **从不**自动创建，要自己 `os.makedirs`。跨轮追问时服务端只把上一轮的 `output/` 与 `uploads/` + 拷进新会话，**`scratch/` 跨轮即丢**。 +- 交付物排序按文件类型（`linsight/domain/utils.py` 的 `_DELIVERABLE_TYPE_RANK`）： + md/docx/pdf/html = 0，xlsx/csv = 1，pptx = 2，未知 = 3，图片 = 4，同档再按 mtime。 + 这是**按类型**不是按轮次：只要 `output/` 里有一个 `.md`，它就**永远**压过 `.xlsx`/`.pptx` 成为用户看到的 + 头条文件 —— 过程稿、提纲、草稿一律放 `scratch/`。 +- 脚本**始终 exit 0，所有诊断走 stdout 的 `print()`**。执行器是二选一：`returncode != 0` 只回 stderr + （stdout 整段丢弃）；`returncode == 0` 只回 stdout（stderr 整段丢弃）。用非零退出码表达失败，报告会整个消失； + 同理，转发子进程输出要 stdout / stderr **各打一行**，`print(r.stdout or r.stderr)` 这种写法会因短路吞掉 stderr。 +- 判是否产出以 **exitcode 0 + 日志**为准。exitcode 0 时执行器会把本轮新建/修改的文件同步进工作区 + （`local_executor.sync_to_workspace`），之后 `ls`/`read_file` 一般能看到，但不要因为一次 `ls` 没看到就重做。 +- 最后两轮工具会收窄到 `write_file`/`edit_file`/`export_*`（`resilience_middleware`）， + 代码执行器被拦掉 —— 交付物要尽早产出，不要拖到收尾。 + +> ⚠️ **E2B 沙箱模式下技能包结构性不可见 —— 三个内置包共同的失效前提。** +> copy-in 的文件清单在工具初始化时快照，早于技能物化（`e2b_executor.py` 的 +> `path_namespace_rules(include_skills=False)`），且 `_materialize_working_set` 目前直接返回空。 +> 结果是沙箱里 `skills/` 整个不存在，调包内脚本必 `FileNotFoundError`；单次执行上限也从本地的 +> 600 秒（`local_executor.py:25`）降到 **300 秒**（`e2b_executor.py:32`）。 +> 依赖包内脚本/模板的技能只在默认的 LocalExecutor 下成立。**SKILL.md 里要给出可执行的自救判定** +> （探一次 `os.path.isdir("skills//scripts")`）**和降级路线**（改走纯库内联写法、把 helper 抄进构建脚本、 +> 自检改为自己核对并告知用户），否则模型只会在 `FileNotFoundError` 上反复重试直到耗尽轮次。 + +**自检脚本（三个内置包共用的约定）** + +- 双段输出：`=== 内容 ===`（读出来给模型看的正文）+ `=== 体检 ===`（Finding）。Finding 三级 + ERROR / WARN / INFO，每条 message 必须带「现象（带具体数字）→ 怎么改」，只有 ERROR 阻塞交付。 +- 结尾固定 `合计: x ERROR / y WARN / z INFO` + 终止串 **`结论: 通过`**（SKILL.md 拿它当模型的停止条件， + 字面不许改）。同一个包里如果有第二个脚本也打这串（如 xlsx 的重算脚本），SKILL.md 必须把两步的先后写清楚。 +- **检查器必须可满足**：阈值按「明显坏掉」定，不要照抄写作规范。规范允许的写法（如 design-zh 允许的 + 10pt 注释字号）若被判 ERROR，一份完全合规的稿子也永远到不了「结论: 通过」，模型就会学会无视整份报告。 + 阈值刻意比规范松一档时，在常量旁写明对应条款，并在报告结尾点明「没报 ERROR ≠ 符合规范」。 +- 脚本崩溃也要 exit 0：主体抽成函数，`main()` 里 `except Exception` → 打 `[FATAL]` + + `traceback.print_exc(file=sys.stdout)` + 一句「怎么办」。损坏/加密/改过扩展名的文件都会走这条路， + 非零退出会让整份报告被执行器丢弃，模型看到的是空白。 +- 脚本的模块 docstring 与代码注释用**英文**，`print` 出去的文案用**中文**；祈使句， + 不写「可能 / 建议尝试」这类软话（模型会当成可选项）。 + +## 待办：灵思交付物的 .pptx 在线预览（已评估，未实施） + +当前 `.pptx` 交付物在前端是 `unsupported`，只能「下载后查看」 +（`client/src/components/Linsight/Artifacts/artifactUtils.ts` 的 `DOCUMENT_EXTS` 不含 pptx； +答案正文里的 `.pptx` 链接也不会被解析成可点预览）。**只有 pptx 有这个问题** —— +`.docx`/`.xlsx`/`.csv` 都在 `DOCUMENT_EXTS` 里，前端用 mammoth / xlsx 直接渲染，无需后端转换。 + +**把 pptx 加进 `DOCUMENT_EXTS` 这一行远远不够。** `FilePreview` 对 ppt/pptx 的策略是复用 PDF viewer +（`pages/knowledge/FilePreview/viewers/index.ts` 里 `pptx: "pdf"`），前提是后端给出一个已经转好的 `preview_url`。 +知识库能做到，是因为它在入库解析阶段就生成了预览 PDF 并把对象名持久化在 `KnowledgeFile.preview_file_object_name` +（见 `knowledge_service.get_file_share_url`）。灵思交付物**完全没有这套**：只有 +`linsight/final_result/{svid}/{file_id}{ext}` 一个对象和 `output_result["final_files"]` 里的元数据， +既没有预览字段，也没有转换时机；`POST /workbench/file_download` 只是把 object key 换成预签名 URL。 + +要做的话需要补齐： + +1. 后端一个转换服务：复用现成的 `knowledge/rag/pipeline/loader/utils/libreoffice_converter.convert_ppt_to_pdf()`， + 把结果缓存到 `linsight/final_result_preview/{svid}/{file_id}.pdf`。 +2. 转换时机建议**首次请求时懒转 + 缓存**（收割时同步转会给每个任务收尾加上 LibreOffice 的 10–30s 冷启动， + 而多数交付物用户并不会点开预览；异步 Celery 方案还要额外维护状态位）。需要处理并发去重与失败降级。 +3. 一个返回 `preview_url` 的端点（或在现有 `file_download` 上加 `preview=true` 分支），沿用同一套分享链接鉴权。 +4. 前端：`DOCUMENT_EXTS` 加 `pptx`、`PreviewBody` 取 `preview_url`、转换失败时回落到现在的「下载后查看」。 + +**工作量估计 ~2–2.5 人天**（后端 1–1.5、前端 0.5、测试 0.5）。 + +**风险（产品决策点）**：服务端只有文泉驿正黑一种中文字体，转出来的 PDF 版式与用户本机 PowerPoint 打开的效果 +会有出入 —— 预览越像"最终稿"，这种不一致越容易变成投诉。真要上，预览页需要明确标注 +「预览为服务端渲染，实际排版以 PowerPoint 打开为准」。 + +结论：**建议单独立项**，不随技能包一起做。 diff --git a/scripts/pack_linsight_skill.sh b/scripts/pack_linsight_skill.sh new file mode 100644 index 0000000000..6dc89dfa0a --- /dev/null +++ b/scripts/pack_linsight_skill.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Package a Linsight skill bundle into an importable .zip. +# +# bash scripts/pack_linsight_skill.sh src/backend/bisheng/linsight/builtin_skills/bisheng-pptx [outdir] +# +# Validates the constraints the backend enforces on import (SKILL.md at the +# archive root, kebab-case name matching the directory, size caps) so failures +# surface here rather than as an 11051/11052/11059 error in the admin UI. + +set -euo pipefail + +SRC="${1:-}" +OUT_DIR="${2:-dist}" + +if [ -z "${SRC}" ]; then + echo "usage: bash scripts/pack_linsight_skill.sh [outdir]" >&2 + exit 2 +fi + +SRC="${SRC%/}" +NAME="$(basename "${SRC}")" + +if [ ! -f "${SRC}/SKILL.md" ]; then + echo "[FAIL] ${SRC}/SKILL.md not found — SKILL.md must sit at the bundle root" >&2 + exit 1 +fi + +# frontmatter name must equal the directory name (deepagents resolves skills by path) +FM_NAME="$(awk '/^---[[:space:]]*$/{n++; next} n==1 && /^name:/{sub(/^name:[[:space:]]*/, ""); gsub(/["\r]/, ""); print; exit}' "${SRC}/SKILL.md")" +if [ -z "${FM_NAME}" ]; then + echo "[FAIL] SKILL.md frontmatter has no 'name'" >&2 + exit 1 +fi +if [ "${FM_NAME}" != "${NAME}" ]; then + echo "[FAIL] frontmatter name '${FM_NAME}' != directory name '${NAME}'" >&2 + exit 1 +fi +if ! printf '%s' "${NAME}" | grep -Eq '^[a-z0-9]+(-[a-z0-9]+)*$'; then + echo "[FAIL] '${NAME}' is not kebab-case; import would silently rewrite it" >&2 + exit 1 +fi + +mkdir -p "${OUT_DIR}" +ZIP_PATH="${OUT_DIR}/${NAME}.zip" +rm -f "${ZIP_PATH}" + +ABS_ZIP="$(cd "${OUT_DIR}" && pwd)/${NAME}.zip" +( + cd "${SRC}" + zip -Xrq "${ABS_ZIP}" . \ + -x '*/__pycache__/*' '__pycache__/*' '*.pyc' '.DS_Store' '*/.DS_Store' +) + +ZIP_BYTES=$(wc -c < "${ZIP_PATH}" | tr -d ' ') +RAW_BYTES=$(find "${SRC}" -type f -not -path '*/__pycache__/*' -not -name '*.pyc' -not -name '.DS_Store' -exec wc -c {} + | tail -1 | awk '{print $1}') +FILE_COUNT=$(find "${SRC}" -type f -not -path '*/__pycache__/*' -not -name '*.pyc' -not -name '.DS_Store' | wc -l | tr -d ' ') + +MAX_ZIP=$((10 * 1024 * 1024)) +MAX_RAW=$((100 * 1024 * 1024)) + +printf '%s\n' "[OK] ${ZIP_PATH}" +printf ' name : %s\n' "${NAME}" +printf ' files : %s\n' "${FILE_COUNT}" +printf ' zip size : %s bytes (limit %s)\n' "${ZIP_BYTES}" "${MAX_ZIP}" +printf ' unpacked : %s bytes (limit %s)\n' "${RAW_BYTES}" "${MAX_RAW}" + +STATUS=0 +if [ "${ZIP_BYTES}" -gt "${MAX_ZIP}" ]; then + echo "[FAIL] zip exceeds the 10MB upload cap (error 11052)" >&2 + STATUS=1 +fi +if [ "${RAW_BYTES}" -gt "${MAX_RAW}" ]; then + echo "[FAIL] unpacked bundle exceeds the 100MB cap (error 11059)" >&2 + STATUS=1 +fi + +if [ "${STATUS}" -eq 0 ]; then + echo " 导入方式: 管理端 → 灵思 → 技能 → 上传，选择上面的 zip（需要租户管理员权限）" +fi +exit "${STATUS}" diff --git a/src/backend/bisheng/core/database/alembic/AGENTS.md b/src/backend/bisheng/core/database/alembic/AGENTS.md index 3f91011a6e..07df47cbd1 100644 --- a/src/backend/bisheng/core/database/alembic/AGENTS.md +++ b/src/backend/bisheng/core/database/alembic/AGENTS.md @@ -142,6 +142,19 @@ grandfathered: do **not** edit released migrations, but never add new ones like constraints. Keep names stable so schema inspection remains understandable. - Identifiers come back **uppercase** from DM8 reflection — compare case-insensitively (the `*_exists` helpers already do). +- **Charset and collation follow the database defaults.** New migrations must not + normally pass MySQL-specific table options such as `mysql_charset` or + `mysql_collate` to `op.create_table()`. Charset and collation are deployment-level + database policy; hard-coding them in one migration makes schemas differ by creation + path, embeds MySQL-only semantics in a MySQL/DM8 codebase, and may change sorting or + uniqueness behaviour unexpectedly. Keep the corresponding SQLModel `__table_args__` + consistent with this rule so `create_all()` and Alembic do not create different table + definitions. An explicit table charset/collation is allowed only when it is a + documented data-contract requirement that cannot safely inherit the database default; + explain the exception in the migration and review its DM8 behaviour. A MySQL client + connection charset (for example `charset=utf8mb4`) controls transport encoding and + does not establish the database or table default. Do not edit an already-released + migration merely to remove legacy charset/collation options. - **Reusable DDL guards** go in `alembic_helpers/online.py` (`table_exists`, `column_exists`) so revisions stay thin. Do **not** treat `alembic_helpers/f011.py` as a template — it holds read-then-write *data* logic from a pre-rule revision, which diff --git a/src/backend/bisheng/core/openfga/client.py b/src/backend/bisheng/core/openfga/client.py index ad3ce8138e..f52dd2c937 100644 --- a/src/backend/bisheng/core/openfga/client.py +++ b/src/backend/bisheng/core/openfga/client.py @@ -268,6 +268,12 @@ async def read_tuples( ) -> list[dict]: """Read tuples matching the given filter. + Pass no filter at all to walk the whole Store. Any other combination must + satisfy the Read API: the object type is mandatory, and the object id and + the user cannot both be empty. Callers used to learn this the hard way — + the server answers a filter it dislikes with a generic validation_error, + which surfaced as a 500 only after the surrounding work had already run. + Returns list of {"key": {"user": ..., "relation": ..., "object": ...}, "timestamp": ...}. """ tuple_key: dict[str, str] = {} @@ -277,6 +283,12 @@ async def read_tuples( tuple_key["relation"] = relation if object: tuple_key["object"] = object + if tuple_key: + object_type, separator, object_id = (object or "").partition(":") + if not object_type or not separator: + raise ValueError(f"OpenFGA read filter needs an object type, got object={object!r}") + if not object_id and not user: + raise ValueError("OpenFGA read filter needs an object id or a user, got neither") tuples: list[dict] = [] continuation_token: str | None = None while True: diff --git a/src/backend/bisheng/department/domain/services/department_service.py b/src/backend/bisheng/department/domain/services/department_service.py index c549cd7d11..1e99a02a6d 100644 --- a/src/backend/bisheng/department/domain/services/department_service.py +++ b/src/backend/bisheng/department/domain/services/department_service.py @@ -1235,6 +1235,31 @@ async def aget_path_tree(cls, login_user, dept_id: int, include_archived: bool = roots = await cls._abuild_pruned_forest([target], {target.id}, is_sys_admin, admin_paths, include_archived) return {"roots": roots, "total_matches": 1, "truncated": False} + @classmethod + async def abuild_forest_within_subtree( + cls, + seeds, + matched_ids: set[int], + *, + confined_to_path: str | None = None, + include_archived: bool = False, + ) -> list[dict]: + """The same pruned-forest assembly, for a caller whose scope is a + department subtree rather than an administrator's scope. + + The grant-subject pickers (F048) ask "who may be granted this resource", + so their visibility is the resource's own department binding — not who + the caller administers. ``confined_to_path=None`` means the whole tenant. + """ + + return await cls._abuild_pruned_forest( + seeds, + matched_ids, + confined_to_path is None, + {confined_to_path} if confined_to_path else set(), + include_archived, + ) + @classmethod async def _abuild_pruned_forest( cls, seeds, matched_ids: set[int], is_sys_admin: bool, admin_paths: set[str], include_archived: bool diff --git a/src/backend/bisheng/knowledge/domain/services/knowledge_permission_service.py b/src/backend/bisheng/knowledge/domain/services/knowledge_permission_service.py index 8ed13ce452..aadafaff43 100644 --- a/src/backend/bisheng/knowledge/domain/services/knowledge_permission_service.py +++ b/src/backend/bisheng/knowledge/domain/services/knowledge_permission_service.py @@ -345,6 +345,58 @@ async def remove_ordinary_sources(self, **kwargs): ... async def sync_public_reader(self, **kwargs): ... +_CONTAINER_KINDS = { + "knowledge_space": {"SPACE"}, + "knowledge_library": {"NORMAL", "QA"}, +} + + +def _container_rejection( + record: KnowledgeContainerPermissionRecord | None, + actor: PermissionActor, + resource_type: str, + resource_id: str, + *, + allowed_statuses: set[str], +) -> str | None: + """Name why a container cannot serve as a permission target, or None. + + The reason never reaches the caller: `permission_error_response` flattens + every rejection into the same 19003 body so that a missing resource and one + in another tenant stay indistinguishable. It goes to the log instead — + without it, "Invalid resource type or ID" covers seven different causes, and + a space stuck at status=FAILED reads exactly like a typo in the id. + """ + + if record is None: + return "NOT_FOUND" + if resource_type not in _CONTAINER_KINDS: + return "UNSUPPORTED_RESOURCE_TYPE" + if record.resource_type != resource_type or record.resource_id != resource_id: + return "IDENTITY_MISMATCH" + if record.kind not in _CONTAINER_KINDS[resource_type]: + return f"KIND_MISMATCH:{record.kind}" + if record.status not in allowed_statuses: + return f"STATUS_NOT_USABLE:{record.status}" + if record.tenant_id != actor.current_tenant_id and not actor.super_admin: + return "TENANT_MISMATCH" + return None + + +def _reject_container( + reason: str, + resource_type: str, + resource_id: str, +) -> PermissionInvalidResourceError: + logger.warning( + "permission target rejected: reason=%s resource=%s:%s", + reason, + resource_type, + resource_id, + ) + return PermissionInvalidResourceError() + + class F048KnowledgeContainerPermissionAdapter: """Keep container business validation outside the permission domain.""" @@ -639,20 +691,15 @@ def _target( resource_type: str, resource_id: str, ) -> VerifiedPermissionTarget: - expected_kinds = { - "knowledge_space": {"SPACE"}, - "knowledge_library": {"NORMAL", "QA"}, - } - if ( - record is None - or resource_type not in expected_kinds - or record.resource_type != resource_type - or record.resource_id != resource_id - or record.status != "PUBLISHED" - or record.kind not in expected_kinds[resource_type] - or (record.tenant_id != actor.current_tenant_id and not actor.super_admin) - ): - raise PermissionInvalidResourceError() + reason = _container_rejection( + record, + actor, + resource_type, + resource_id, + allowed_statuses={"PUBLISHED"}, + ) + if reason is not None: + raise _reject_container(reason, resource_type, resource_id) return VerifiedPermissionTarget.from_business_service( tenant_id=record.tenant_id, resource_type=record.resource_type, @@ -668,20 +715,15 @@ def _lifecycle_target( resource_type: str, resource_id: str, ) -> VerifiedPermissionTarget: - expected_kinds = { - "knowledge_space": {"SPACE"}, - "knowledge_library": {"NORMAL", "QA"}, - } - if ( - record is None - or resource_type not in expected_kinds - or record.resource_type != resource_type - or record.resource_id != resource_id - or record.status not in {state.name for state in KnowledgeState} - or record.kind not in expected_kinds[resource_type] - or (record.tenant_id != actor.current_tenant_id and not actor.super_admin) - ): - raise PermissionInvalidResourceError() + reason = _container_rejection( + record, + actor, + resource_type, + resource_id, + allowed_statuses={state.name for state in KnowledgeState}, + ) + if reason is not None: + raise _reject_container(reason, resource_type, resource_id) return VerifiedPermissionTarget.from_business_service( tenant_id=record.tenant_id, resource_type=record.resource_type, diff --git a/src/backend/bisheng/knowledge/domain/services/knowledge_service.py b/src/backend/bisheng/knowledge/domain/services/knowledge_service.py index 59e31cbe24..b86e25e0f4 100644 --- a/src/backend/bisheng/knowledge/domain/services/knowledge_service.py +++ b/src/backend/bisheng/knowledge/domain/services/knowledge_service.py @@ -491,9 +491,8 @@ async def get_knowledge( batch_action_map = await cls.permission_service.get_knowledge_action_map_async( login_user, [int(one.id) for one in batch], - _KNOWLEDGE_LIST_ACTIONS, + [action], ) - action_map.update(batch_action_map) authorized.extend(one for one in batch if action in batch_action_map.get(int(one.id), set())) if len(batch) < _KNOWLEDGE_PERMISSION_SCAN_BATCH_SIZE: break @@ -515,9 +514,8 @@ async def get_knowledge( batch_action_map = await cls.permission_service.get_knowledge_action_map_async( login_user, [int(one.id) for one in batch], - _KNOWLEDGE_LIST_ACTIONS, + [action], ) - action_map.update(batch_action_map) res.extend(one for one in batch if action in batch_action_map.get(int(one.id), set())) if len(res) >= fetch_limit or len(batch) < _KNOWLEDGE_PERMISSION_SCAN_BATCH_SIZE: break @@ -542,6 +540,17 @@ async def get_knowledge( if has_more: res = res[:page_size] + # The other list actions only decorate the rows that survived, so they + # are resolved once the page is known. Asking for all of them per + # candidate multiplied the scan by the number of actions, and that cost + # grew with every extra scan round instead of with the page. + if res: + action_map = await cls.permission_service.get_knowledge_action_map_async( + login_user, + [int(one.id) for one in res], + _KNOWLEDGE_LIST_ACTIONS, + ) + # ---- 4. Enrich + build response ---- enrich_start = perf_counter() result_data = await cls.aconvert_knowledge_read( diff --git a/src/backend/bisheng/knowledge/rag/pipeline/loader/excel.py b/src/backend/bisheng/knowledge/rag/pipeline/loader/excel.py index 1d5e86b22f..88fabf3bb7 100644 --- a/src/backend/bisheng/knowledge/rag/pipeline/loader/excel.py +++ b/src/backend/bisheng/knowledge/rag/pipeline/loader/excel.py @@ -1,10 +1,15 @@ import os +import re from langchain_core.documents import Document from bisheng.common.constants.knowledge import KNOWLEDGE_MAX_CHUNK_CHARS from bisheng.common.errcode.knowledge import KnowledgeExcelChunkMaxError from bisheng.knowledge.rag.pipeline.loader.base import BaseBishengLoader +from bisheng.knowledge.rag.pipeline.loader.utils.excel_images import ( + ExcelImage, + extract_excel_images, +) from bisheng.knowledge.rag.pipeline.loader.utils.md_from_excel import ( ExcelRowTooLongError, convert_file_to_markdown, @@ -18,6 +23,16 @@ DEFAULT_CHUNK_SIZE = 1000 DEFAULT_CHUNK_OVERLAP = 100 +# Only the OPC package carries drawings. Legacy .xls is an OLE container, and +# .et is handled by the xinchuang delegate loader before it reaches markdown. +IMAGE_CAPABLE_EXTENSIONS = frozenset({"xlsx"}) + + +def _safe_media_name(name: str) -> str: + """Keep the staged file name MinIO/URL safe, per the BaseBishengLoader contract.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]", "_", os.path.basename(name)) + return cleaned or "image" + class ExcelLoader(BaseBishengLoader): def __init__( @@ -71,6 +86,66 @@ def _split_long_row_text(self, text: str, budget: int) -> list[str]: ) return splitter.split_text(text) + def _build_document(self, content: str, chunk_index: int) -> Document: + metadata = self.file_metadata.copy() + metadata["chunk_index"] = chunk_index + metadata["bbox"] = "" + metadata["page"] = 0 + return Document(page_content=content, metadata=metadata) + + def _emit_image_documents(self, images: list[ExcelImage], start_chunk_index: int) -> list[Document]: + """Stage embedded pictures and turn them into their own markdown chunks. + + Pictures live outside the cell grid and cannot be folded into the table + markdown: a sheet holding nothing but a drawing (report exporters do + this) produces no markdown at all, and the renderer's sheet numbering + skips empty sheets, so there is no chunk to attach them to. Each sheet's + pictures therefore become their own chunk, captioned with the sheet name + so the segment still carries some retrievable context. + + Only the bytes are staged on local disk here; ImageUploadTransformer + performs the MinIO upload, per the image contract in BaseBishengLoader. + """ + if not images: + return [] + + image_dir = self.ensure_local_image_dir() + by_sheet: dict[str, list[ExcelImage]] = {} + for image in images: + by_sheet.setdefault(image.sheet_name, []).append(image) + + staged: set[str] = set() + documents: list[Document] = [] + chunk_index = start_chunk_index + + for sheet_name, sheet_images in by_sheet.items(): + heading = f"## {sheet_name}" + refs: list[str] = [] + for image in sheet_images: + filename = _safe_media_name(image.media_name) + # The same media part may be anchored on several sheets; stage once. + if filename not in staged: + with open(os.path.join(image_dir, filename), "wb") as f: + f.write(image.content) + staged.add(filename) + refs.append(f"![{filename}]({self.build_image_url(filename)})") + + block = heading + for ref in refs: + candidate = f"{block}\n\n{ref}" + # Keep the heading on every chunk when a sheet has enough pictures + # to overflow one segment. + if len(candidate) > self.max_chunk_limit and block != heading: + documents.append(self._build_document(block, chunk_index)) + chunk_index += 1 + block = f"{heading}\n\n{ref}" + else: + block = candidate + documents.append(self._build_document(block, chunk_index)) + chunk_index += 1 + + return documents + def load(self) -> list[Document]: if os.path.exists(self.file_path): self.preview_file_path = self.file_path @@ -100,14 +175,13 @@ def load(self) -> list[Document]: full_file_name = f"{md_file_path}/{file_name}" with open(full_file_name, encoding="utf-8") as f: content = f.read() - one_metadata = self.file_metadata.copy() - one_metadata["chunk_index"] = chunk_index - one_metadata["bbox"] = "" - one_metadata["page"] = 0 # Defensive: the char budget above should have prevented this. If it # ever fires, the budget math has drifted from the renderer. if len(content) > self.max_chunk_limit: raise KnowledgeExcelChunkMaxError() - documents.append(Document(page_content=content, metadata=one_metadata)) + documents.append(self._build_document(content, chunk_index)) + + if self.file_extension.lower().lstrip(".") in IMAGE_CAPABLE_EXTENSIONS: + documents.extend(self._emit_image_documents(extract_excel_images(self.file_path), len(documents))) return documents diff --git a/src/backend/bisheng/knowledge/rag/pipeline/loader/utils/excel_images.py b/src/backend/bisheng/knowledge/rag/pipeline/loader/utils/excel_images.py new file mode 100644 index 0000000000..623d2b50c6 --- /dev/null +++ b/src/backend/bisheng/knowledge/rag/pipeline/loader/utils/excel_images.py @@ -0,0 +1,163 @@ +"""Extract pictures embedded in an xlsx workbook, grouped by the sheet they sit on. + +openpyxl is not used here: it only exposes images through the private +``worksheet._images`` and drops anchors it does not understand. Reading the OPC +package directly is both complete and cheap. + +A picture belongs to exactly one sheet, and that ownership is only discoverable +through the relationship chain: + + workbook.xml + -> workbook.xml.rels rId -> worksheets/sheetN.xml + -> sheetN.xml.rels drawing -> drawings/drawingM.xml + -> drawingM.xml.rels r:embed -> media/imageX.png + +Sheet order and file numbering are unrelated (deleting a sheet leaves gaps in +sheetN.xml), so the chain must be followed instead of pairing them positionally. +""" + +import posixpath +import zipfile +from dataclasses import dataclass +from xml.etree import ElementTree + +from loguru import logger + +RELS_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +IMAGE_EXTENSIONS = frozenset({"png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif", "jfif"}) + + +@dataclass +class ExcelImage: + """One embedded picture and the sheet it is anchored to.""" + + sheet_name: str + # Media part name inside the package, e.g. "image1.png". Unique per workbook. + media_name: str + ext: str + content: bytes + + +def _local_name(tag: str) -> str: + """Strip the ``{namespace}`` prefix ElementTree keeps on every tag.""" + return tag.rsplit("}", 1)[-1] + + +def _find_all(root: ElementTree.Element, local_name: str) -> list[ElementTree.Element]: + """Find descendants by local name — OOXML namespace prefixes are arbitrary.""" + return [el for el in root.iter() if _local_name(el.tag) == local_name] + + +def _resolve(base_dir: str, target: str) -> str: + """Resolve a relationship target (often "../drawings/x.xml") to a package path.""" + if target.startswith("/"): + return target.lstrip("/") + return posixpath.normpath(posixpath.join(base_dir, target)) + + +def _rels_path(part_path: str) -> str: + base_dir, name = posixpath.split(part_path) + return posixpath.join(base_dir, "_rels", f"{name}.rels") + + +def _read_xml(zf: zipfile.ZipFile, path: str) -> ElementTree.Element | None: + try: + return ElementTree.fromstring(zf.read(path)) + except KeyError: + return None + except ElementTree.ParseError: + logger.warning("excel image extraction: malformed xml part {}", path) + return None + + +def _read_relationships(zf: zipfile.ZipFile, part_path: str) -> dict[str, tuple[str, str]]: + """Map rId -> (relationship type, resolved package path).""" + root = _read_xml(zf, _rels_path(part_path)) + if root is None: + return {} + + base_dir = posixpath.dirname(part_path) + rels: dict[str, tuple[str, str]] = {} + for rel in _find_all(root, "Relationship"): + rel_id = rel.get("Id") + target = rel.get("Target") + if not rel_id or not target or rel.get("TargetMode") == "External": + continue + rels[rel_id] = (rel.get("Type") or "", _resolve(base_dir, target)) + return rels + + +def _drawing_media_paths(zf: zipfile.ZipFile, drawing_path: str) -> list[str]: + """Package paths of the media referenced by one drawing, in document order.""" + root = _read_xml(zf, drawing_path) + if root is None: + return [] + + rels = _read_relationships(zf, drawing_path) + paths: list[str] = [] + for blip in _find_all(root, "blip"): + rel_id = blip.get(f"{{{RELS_NS}}}embed") + if not rel_id or rel_id not in rels: + continue + # The same picture may be anchored more than once on a sheet; one copy is enough. + media_path = rels[rel_id][1] + if media_path not in paths: + paths.append(media_path) + return paths + + +def extract_excel_images(xlsx_path: str) -> list[ExcelImage]: + """Return every embedded picture together with its owning sheet. + + Returns an empty list for anything that is not a readable xlsx package + (legacy .xls is an OLE container, not a zip) — callers treat picture + extraction as best effort and must not fail the file over it. + """ + try: + with zipfile.ZipFile(xlsx_path) as zf: + return _extract(zf) + except (zipfile.BadZipFile, OSError) as e: + logger.warning("excel image extraction skipped for {}: {}", xlsx_path, e) + return [] + + +def _extract(zf: zipfile.ZipFile) -> list[ExcelImage]: + workbook = _read_xml(zf, "xl/workbook.xml") + if workbook is None: + return [] + + workbook_rels = _read_relationships(zf, "xl/workbook.xml") + images: list[ExcelImage] = [] + + for sheet_el in _find_all(workbook, "sheet"): + sheet_name = sheet_el.get("name") + rel_id = sheet_el.get(f"{{{RELS_NS}}}id") + if not sheet_name or not rel_id or rel_id not in workbook_rels: + continue + + sheet_path = workbook_rels[rel_id][1] + drawing_rels = [ + target for rel_type, target in _read_relationships(zf, sheet_path).values() if rel_type.endswith("/drawing") + ] + + for drawing_path in drawing_rels: + for media_path in _drawing_media_paths(zf, drawing_path): + media_name = posixpath.basename(media_path) + ext = media_name.rsplit(".", 1)[-1].lower() if "." in media_name else "" + if ext not in IMAGE_EXTENSIONS: + continue + try: + content = zf.read(media_path) + except KeyError: + logger.warning("excel image extraction: missing media part {}", media_path) + continue + images.append( + ExcelImage( + sheet_name=sheet_name, + media_name=media_name, + ext=ext, + content=content, + ) + ) + + return images diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/SKILL.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/SKILL.md new file mode 100644 index 0000000000..1414f7b170 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/SKILL.md @@ -0,0 +1,239 @@ +--- +name: bisheng-docx +description: >- + 制作或修改 Word 文档（.docx）时使用：撰写工作报告、分析报告、方案、纪要、通知、公文、 + 合同初稿、说明书、简历、周报月报等正式文档；把已有内容整理成带标题层级、目录、页码、 + 表格和图表的 Word 文件；读取、改写用户上传的 .docx。当用户说到「Word」「文档」「报告」 + 「方案」「纪要」「公文」「通知」「说明书」「docx」「写一份 X」「整理成 Word」 + 「导出成 Word」时触发。本技能给出 BiSheng 代码执行器里唯一可行的 python-docx 路径 + （该环境没有 Node/docx-js，中文字体必须显式写 w:eastAsia 才生效）， + 以及中文排版规范、目录页码写法、交付前自检与渲染脚本。 + 只是把已经写好的回答原样存成 Word、不要求目录页码表格版式的，用 export_docx 更省事。 + 如果用户要的是 Excel 表格、PPT 或纯 Markdown，不要用本技能。 +metadata: + display-name: Word 文档制作（BiSheng 适配） +--- + +# 在 BiSheng 里做 DOCX + +## 0. 开工纪律 + +**这一轮只读文档，不要在同一轮里并行调用别的工具。** 读完本文件（必要时再读 references）之后， +下一轮才开始动手。把「读 SKILL.md」和「产出交付物」放进同一轮并行调用，等于技能没读。 + +**本技能要求已勾选代码执行器（`bisheng_code_interpreter`）。** 没有它就无法生成 .docx —— +直接告诉用户「请在工具里勾选代码执行器后重试」。 + +> 灵思自带 `export_docx`（markdown → Word）。**两者分工是硬的，选错路线是本技能最大的失败风险 +> —— 动手前先看 §2 的边界表。** 一句话：只要「把刚才的内容存成 Word」用 `export_docx`； +> 要目录、页码、A4 版式、表格列宽、封面、或改用户上传的 .docx，才用本技能。 + +## 1. 环境事实（照做，不要试探） + +| 项 | 事实 | +|---|---| +| **生成方式** | **只有 `python-docx`**（后端 `pyproject.toml` 的正式依赖），`import docx` 直接可用 | +| **中文字体** | `run.font.name` 只管西文；中文必须写 `w:eastAsia`。用本技能包的 `set_run_font()` | +| 不存在的东西 | Node/npm/`docx-js`、`markitdown`、`defusedxml`、`pdftoppm`、`zip`/`unzip` 命令 | +| 可能没有 | `pandoc`（发布镜像有，手工部署的机器未必）、`soffice`（只影响可选的渲染预览） | +| 其它可用库 | pandas/numpy、Pillow、PyMuPDF(`fitz`)、matplotlib、openpyxl、python-pptx、lxml | +| 禁止 | `pip install`、`npm install`、任何联网假设（生产多为离线内网） | +| 工作目录 | 执行器 cwd = 工作区根，**一律用相对路径** | +| `output/` | 唯一交付区 | +| `scratch/` | 中间产物区，**不会交付**，需自己 `os.makedirs` | +| `uploads/` | 用户上传的原件在这里 | +| `skills/bisheng-docx/` | 本技能包，脚本和参考资料在这里，只读 | +| 绝对禁止 | 写 `/output/xxx.docx` 这种带前导斜杠的路径 —— 文件被静默丢弃，用户拿不到 | +| 单次执行上限 | 本地执行器 **600 秒**，E2B 沙箱 **300 秒**。构建、体检、渲染分三次调用，不要挤在一次里 | +| E2B 沙箱 | 该模式下 `skills/` **结构性不可见**（文件快照早于技能物化）→ 调包内脚本必报 `FileNotFoundError`。出现这个报错就别再试包内脚本，把 cookbook §1 的 `w:eastAsia` 五行 XML 抄进构建脚本自己写 | +| 日志规则 | 执行器二选一：**退出码 0 只回 stdout，非 0 只回 stderr** → 一切诊断用 `print()`，子进程的 stdout/stderr 分两行都打出来 | +| 代码围栏 | 提交给执行器的代码里只要出现三反引号围栏，就**只执行围栏里的内容** —— 别在代码参数里贴 markdown | +| 可见性 | 退出码 0 时执行器会把本轮新增/修改的文件同步进工作区，之后 `ls`/`read_file` 一般能看到。判成功以 **退出码 0 + 日志**为准，不要反复找文件、更不要重做一遍 | +| 交付物排序 | `.md` 与 `.docx` **同为最高档**，同档比修改时间 → 后写的 `.md` 会直接压过 `.docx` 成为用户看到的头条。过程稿一律 `scratch/` | +| 轮次 | 最后两轮代码执行器会被摘除 → 文档要尽早产出，不要拖到收尾 | + +### 1.1 先探一次环境（第一次执行代码时顺手做，只花一轮） + +```python +import shutil +try: + import docx + print("python-docx OK") +except ImportError: + print("python-docx MISSING") +print("soffice:", shutil.which("soffice") or "无（只影响可选的渲染预览，不影响生成）") +print("pandoc :", shutil.which("pandoc") or "无（读 .docx 用 inspect_docx.py 即可）") +``` + +`python-docx MISSING` 正常部署不会出现。**不要 `pip install`**（共享的离线环境，装了会污染所有租户）， +如实告诉用户环境缺依赖、需要运维补装。 + +## 2. 选路线（先划清与 `export_docx` 的边界） + +`export_docx` 走的是 MarkDocx：读 `output/` 下的 markdown，纯 python-docx 转成 Word。 +下面是它的实测产物特征 —— 对照着选，别凭感觉： + +| 维度 | `export_docx` | 本技能 | +|---|---|---| +| 输入 | 只吃 `output/` 下的 **markdown**，单向 | 直接写 .docx，也能读改已有 .docx | +| 纸张 | **US Letter 21.6×27.9cm**（不是 A4），左右边距 3.2cm，改不了 | A4 纵/横随选，边距可调 | +| 标题 | 用内置 Heading 样式，中文字体已设（黑体/楷体） | 同样用内置 Heading，字体字号全可控 | +| 目录 / 页码 | **都没有**，也加不了 | `add_toc` + `add_page_number_footer` | +| 表格 | 宽度 auto，无表头底纹、无列宽、不重复表头 | 列宽写到每个单元格、表头深底白字、跨页重复表头 | +| 图片 | 固定 5.7 英寸宽 | 按版心等比缩放 + 图注 | +| 封面、分节、页眉、分隔线 | 没有 | 都有 | +| 代价 | 一次调用，零风险 | 写脚本 + 体检，2–4 轮 | + +| 情况 | 做法 | +|---|---| +| 用户只要「把刚才的内容存成 Word」，没提版式要求 | `export_docx`，**不要用本技能** | +| 用户要目录 / 页码 / A4 / 表格列宽 / 封面 / 特定字体字号 | §3 从零构建 | +| 用户上传了 .docx 要改 | §5 改已有文件（`export_docx` 做不到） | +| 用户上传了模板要按样式填 | 用 `Document("uploads/模板.docx")` 打开**模板本身**，样式自动继承 | +| 拿不准 | 按用户原话里的名词判：出现「正式/对外/打印/汇报稿/公文/目录/页码」走本技能，否则走 `export_docx` | + +## 3. 从零构建 + +**第 1 步 · 定结构**。先把提纲写到 `scratch/outline.md`。 +**提纲、草稿、任何 `.md` 都不许进 `output/`** —— `.md` 和 `.docx` 在交付物排序里同档、按修改时间比， +后写的 `.md` 会顶掉 `.docx` 成为用户看到的头条文件。定清楚几级标题、哪里要表、哪里要图。 + +**第 2 步 · 写构建脚本**。用 `write_file` 把完整脚本写到 `scratch/build_doc.py`， +**不要把整段代码塞进代码执行器的参数里** —— 参数过长会被截断，导致反复重试却总差一截。 +写法读 `/skills/bisheng-docx/references/python-docx-cookbook.md` +（中文字体、标题、目录、页码、表格、图片、列表都有可直接抄的片段）， +排版规范读 `/skills/bisheng-docx/references/design-zh.md`。 + +骨架长这样： + +```python +import sys, os +sys.path.insert(0, "skills/bisheng-docx/scripts") +from docx_helpers import setup_page, apply_chinese_defaults, add_heading_cn, add_body, \ + add_table, add_toc, add_page_number_footer +from docx import Document + +os.makedirs("output", exist_ok=True) +doc = Document() +section = setup_page(doc) # A4 纵向 +apply_chinese_defaults(doc) # ★ 必须调：设好中文字体和标题样式 +add_heading_cn(doc, "标题", 1) +add_toc(doc) # 3 页以上才需要 +add_heading_cn(doc, "一、xxx", 2) +add_body(doc, "正文……") +add_page_number_footer(section) +doc.save("output/xxx.docx") +print("saved") +``` + +**第 3 步 · 执行**： + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "scratch/build_doc.py"], capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +> 为什么不直接写 `python scratch/build_doc.py`：PATH 里的 `python` 未必是后端那个解释器， +> 用 `sys.executable` 才能保证跑在装了 python-docx 的环境里。**下面所有脚本调用都用这个写法。** + +**第 4 步 · 体检并返修**（§6）。返修用 `edit_file` 定点改 `scratch/build_doc.py` 再重跑， +不要每次重写整份脚本。 + +## 4. ★ 中文字体（最容易出的错） + +`run.font.name = "微软雅黑"` **对中文完全无效** —— 它只写 `w:ascii`/`w:hAnsi`（西文）， +中文由 `w:eastAsia` 决定。不设的话中文退回主题字体，用户打开就是"字体乱了"。 + +```python +from docx_helpers import set_run_font +set_run_font(run, "微软雅黑", size_pt=11, bold=True) # ✅ +run.font.name = "微软雅黑" # ❌ 中文不生效 +``` + +`apply_chinese_defaults(doc)` 会把 Normal 和 Heading 1–4 一次性设好（顺带关掉 Word 内置 +Heading 4 的斜体）。用 `add_body` / `add_heading_cn` 写的内容自动就是对的。 +体检脚本专门抓这条。 + +## 5. 改用户上传的文档 + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-docx/scripts/inspect_docx.py", + "uploads/原文.docx", "--content-only"], capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +它按段落序号打印样式和文本、逐表打印内容，据此规划改哪里。然后： + +- **只改需要改的 run**，不要整段重建 —— 重建会丢掉原有字体、编号、批注关联。 +- 原文的样式约定压倒本技能的规范，跟着它走。 +- 改完另存到 `output/`，不要覆盖 `uploads/` 里的原件。 +- Word 会把一句话拆进多个 `w:r`，**看得见的短语在 XML 里未必是连续字符串** —— + 跨 run 的替换要先合并 run，或整段重写后手动补格式。 + +## 6. 交付前体检（必做） + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-docx/scripts/inspect_docx.py", "output/xxx.docx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +输出分两段： + +- **`=== 内容 ===`**：页面尺寸、段落数、逐段样式与文本、逐表内容。用它核对错字、顺序、缺漏。 +- **`=== 体检 ===`**：ERROR 必须修完；WARN 逐条复核（确认无误可交付）；INFO 是建议。 + 覆盖中文字体缺 `w:eastAsia`（含样式继承链）、字号过小、标题层级跳级、表格超出版心、 + 表格首行不像表头、长表不重复表头、图片超宽、目录域没开 updateFields、目录域下没有 Heading、 + 残留占位符、段落里塞 `\n`、连续空段落、长文档缺页码域。 + +结尾固定是 `合计: x ERROR / y WARN / z INFO` 加一行结论。**ERROR 清零就会打出「结论: 通过」**， +看到它才算过关；没到就改 `scratch/build_doc.py` 重新生成再跑一次。不要为了消灭 WARN 无限返工。 + +**可选 · 看渲染图**： + +⚠️ **仅在你确知当前模型支持读图时才做。** 这些 PNG 会被编成 base64 图片块发给厂商接口， +而默认的 Qwen/dashscope 通道已知不接收 base64 图片 —— 读图可能直接失败，甚至中断整个请求。 +拿不准就跳过这一步，以体检结果为准。 + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-docx/scripts/render_docx.py", "output/xxx.docx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +它把每页渲染成 `scratch/preview/<名字>/page-01.png`，再用 `read_file` 逐张查看。 +**渲染用的中文字体只有文泉驿正黑**，和用户 Word 里的实际字体宽度不同 —— +行长松紧只作参考，不要为了预览效果反复微调字号。 +环境里没有 LibreOffice（.docx 走的是 Writer 组件）时脚本会直说，跳过这一步、以体检结果为准即可。 + +## 7. 交付纪律 + +- `output/` 里**只放最终的 `.docx`**。提纲、构建脚本、预览图、任何 `.md` 一律放 `scratch/`。 + **理由是硬的**：交付物排序里 `.md` 与 `.docx` 同为最高档、同档比修改时间， + 而过程稿总是后写的 → 一份 `output/outline.md` 会永久压在 `.docx` 前面当头条， + 用户点开看到的是草稿。`scratch/` 不会交付，但**跨轮追问会丢**（`output/`、`uploads/` 才会带到下一轮）。 +- 文件名用有意义的中文名，如 `output/2024年度经营分析报告.docx`。 +- 收尾时如实说明做了什么、多少页/多少节、有没有目录和页码。 + **不要声称生成了实际不存在的文件。** + +## 8. 绝不要做的事 + +- ❌ 写 `require('docx')` 或任何 Node 脚本 —— 装不上，`npm install` 也会失败。 +- ❌ 跑 `unzip`/`zip` 拆装 .docx —— 环境里没有这两个命令。要改 XML 用 Python 的 `zipfile`。 +- ❌ `pip install` 任何东西。 +- ❌ 给代码执行器的路径带前导斜杠（`/output/...`、`/skills/...`）。**两套命名空间别混**： + `read_file` 读技能文档带斜杠（`/skills/bisheng-docx/references/design-zh.md`）， + 代码执行器里的路径一律不带（`skills/bisheng-docx/scripts/inspect_docx.py`、`output/x.docx`）。 +- ❌ 只写 `run.font.name` 就以为中文字体设好了。 +- ❌ 用手写的 `•` / `1.` 当列表，用 `\n` 当换行。 +- ❌ 用单行表格画分隔线（用 `add_hr`）。 +- ❌ 把文档降级成 Markdown 或 PDF 交付。用户要的是 .docx。 +- ❌ 退出码 0、日志已写明 saved，却因为一时没看到文件就重做一遍。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/references/design-zh.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/references/design-zh.md new file mode 100644 index 0000000000..3787881368 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/references/design-zh.md @@ -0,0 +1,72 @@ +# 中文文档排版规范 + +给「要发出去给人看、可能还要打印」的 Word 文档用。 + +## 1. 页面与字号 + +| 项 | 取值 | +|---|---| +| 纸张 | A4 纵向（21 × 29.7cm）；表格多、列宽的报告可用横向 | +| 页边距 | 上下左右各 2.54cm（公文类可用 上3.7 下3.5 左2.8 右2.6） | +| 正文 | 微软雅黑 / 宋体 11pt，行距 1.5，段后 6pt | +| 一级标题 | 20pt 加粗 | +| 二级标题 | 16pt 加粗 | +| 三级标题 | 14pt 加粗 | +| 四级标题 | 12pt 加粗（**记得关斜体**，Word 内置是斜体） | +| 表格文字 | 10pt | +| 图注表注 | 9pt 灰字，居中 | + +- **正文首行缩进 2 个字符**（`first_line_indent`），不要用空格或 Tab 凑。 +- 中文正文用两端对齐（JUSTIFY），标题左对齐；只有封面和大标题居中。 +- 字体只用**微软雅黑、宋体、黑体、等线**这类用户机器一定有的。渲染在用户那边完成， + 选个冷门字体等于赌它装了。 + +## 2. 结构 + +- 标题**必须用内置 Heading 样式**，否则目录生成不出来、导航窗格是空的。 +- 层级不要跳（H2 直接到 H4 会被体检拦），也不要超过四级。 +- 编号风格全篇统一：中文序号（一、（一）、1.）或阿拉伯（1、1.1、1.1.1）挑一套，不要混。 +- 超过 3 页就加目录；超过 5 页加页码。 +- 用**段前段后间距**分隔内容，不要连打空段落。 + +## 3. 表格 + +- 列宽合计不能超过版心宽度（A4 + 2.54cm 边距 ≈ **15.9cm**），否则右侧列会被截出页面。 + 用 `content_width_cm(section)` 取准确值。 +- 表头深底白字加粗居中；数据行 10pt；隔行浅底（`F2F6FA`）提升可读性。 +- **数字右对齐，文字左对齐**，同一列对齐方式一致。 +- 数字带千分位；单位写在表头（`收入（万元）`），不要每格拼「万元」。 +- 表格上方给一句话说明它在说什么，下方必要时给数据来源。 +- 跨页的长表要重复表头（首行写 `w:tblHeader`，`add_table` 已自动设；手写表格调 `repeat_header_row`）。 +- 表头要么加粗、要么带底纹，两者都没有体检会报 WARN。 + +## 4. 图 + +- 图片按版心宽度等比缩放，不要撑出页面（`add_image_fitted` 已处理）。 +- 每张图配图注（`图 1 xxx`），居中 9pt 灰字。 +- 图表优先用 matplotlib 生成 PNG 再插入；记得设中文字体，否则全是方框。 + +## 5. 语言 + +- 术语全篇统一。用户叫「营收」就别一会儿写「营业收入」一会儿写「主营收入」。 +- 数字、日期、单位格式统一（`2024-01-15` 或 `2024年1月15日`，挑一种）。 +- 中英文之间不用手动加空格（Word 会自动处理），但中文标点要用全角。 +- 不要出现「本文档由 AI 生成」之类的话，除非用户要求。 + +## 6. 交付前 + +跑 `inspect_docx.py`：ERROR 清零到打出「结论: 通过」，WARN 逐条复核。 +`render_docx.py` 渲成图看是**可选**的 —— ⚠️ 仅在你确知当前模型支持读图时才做， +默认的 Qwen/dashscope 通道不接收 base64 图片，读图可能失败甚至中断请求；拿不准就跳过。 +即使能看：**预览的字体和用户 Word 里不一样**（服务端只有文泉驿正黑）， +行长松紧只作参考，不要为了预览效果反复微调字号。 + +## 7. 常见"一眼就知道是机器做的" + +- 标题用手写加粗的普通段落，不是 Heading 样式 → 没有目录、没有导航。 +- 中文字体没设 `w:eastAsia`，正文一半宋体一半雅黑。 +- 表格撑出页面，或所有列等宽把中文列挤成竖排。 +- 用空段落而不是段距来分隔，导致分页后页首出现大片空白。 +- 全篇没有一张图、一个表，纯文字段落堆到底。 +- 四级标题是斜体（Word 默认样式没改）。 +- 段落里塞 `\n` 当换行，结果全挤成一段。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/references/python-docx-cookbook.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/references/python-docx-cookbook.md new file mode 100644 index 0000000000..1af1262a14 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/references/python-docx-cookbook.md @@ -0,0 +1,222 @@ +# python-docx 速查（BiSheng 适配） + +可直接抄的片段。代码假定写在**构建脚本**里（`scratch/build_doc.py`），由代码执行器用 +`subprocess.run([sys.executable, "scratch/build_doc.py"])` 跑。 + +开头统一这样起： + +```python +import sys, os +sys.path.insert(0, "skills/bisheng-docx/scripts") +from docx_helpers import ( + setup_page, apply_chinese_defaults, set_run_font, add_heading_cn, add_body, + add_table, add_toc, add_page_number_footer, add_hr, add_image_fitted, + set_cell_shading, content_width_cm, first_line_indent, CN_FONT, +) +from docx import Document +from docx.shared import Pt, Cm, RGBColor +from docx.enum.text import WD_ALIGN_PARAGRAPH + +os.makedirs("output", exist_ok=True) +os.makedirs("scratch", exist_ok=True) + +doc = Document() +section = setup_page(doc) # A4 纵向，2.54cm 页边距 +apply_chinese_defaults(doc) # ★ 必须调，见下 +``` + +--- + +## 1. ★ 中文字体：最重要的一条 + +**`run.font.name = "微软雅黑"` 对中文完全无效。** 它只写 `w:ascii` 和 `w:hAnsi`（西文）， +中文字符由 `w:eastAsia` 决定；不设的话中文会退回主题字体，和你指定的西文字体对不上， +用户打开一看就是"字体乱了"。 + +```python +set_run_font(run, "微软雅黑", size_pt=11, bold=True, color="1F1F1F") # ✅ 三个都设 +run.font.name = "微软雅黑" # ❌ 中文不生效 +``` + +`apply_chinese_defaults(doc)` 会把 Normal 和 Heading 1–4 的 `w:eastAsia` 一次性设好 +（顺带把 Word 内置 Heading 4 的**斜体**关掉 —— 不关的话四级标题会渲成斜体衬线）。 +之后用 `add_body` / `add_heading_cn` 写的内容就都是对的。 + +体检脚本会专门抓这条。 + +**拿不到 `docx_helpers` 时（E2B 沙箱下 `skills/` 不可见）**，把这五行抄进构建脚本自己写： + +```python +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + +def set_run_font(run, name, size_pt=None, bold=None): + run.font.name = name # 只写 w:ascii / w:hAnsi + rpr = run._element.get_or_add_rPr() + rfonts = rpr.find(qn("w:rFonts")) + if rfonts is None: + rfonts = OxmlElement("w:rFonts"); rpr.insert(0, rfonts) + for attr in ("w:ascii", "w:hAnsi", "w:eastAsia"): + rfonts.set(qn(attr), name) # ★ w:eastAsia 才管中文 + if size_pt is not None: + from docx.shared import Pt; run.font.size = Pt(size_pt) + if bold is not None: + run.font.bold = bold + return run +``` + +样式（Normal / Heading N）用同一段代码，把 `run._element` 换成 `style.element` 即可。 + +## 2. 标题与正文 + +```python +add_heading_cn(doc, "2024 年度经营分析报告", 1) +add_heading_cn(doc, "一、总体情况", 2) +add_body(doc, "本年度公司实现营业收入 12,500 万元，同比增长 15.2%。") # 自动首行缩进 2 字 +add_body(doc, "这段不缩进。", indent=False) +``` + +> **必须用内置 Heading 样式**（`add_heading_cn` 用的就是）。自定义样式的标题不会进目录， +> 导航窗格也是空的。 + +手写段落时： + +```python +p = doc.add_paragraph() +set_run_font(p.add_run("加粗片段"), CN_FONT, 11, bold=True) +set_run_font(p.add_run("，普通片段。"), CN_FONT, 11) +p.alignment = WD_ALIGN_PARAGRAPH.JUSTIFY # 中文正文常用两端对齐 +first_line_indent(p, 2, 11) +``` + +> **绝不要用 `\n`**。python-docx 不会把它变成换行，要么用多个 `Paragraph`， +> 要么 `run.add_break()`。 + +## 3. 目录与页码 + +```python +add_toc(doc, levels="1-3") # 插入目录域 + 打开 updateFields +add_page_number_footer(section) # 页脚「第 X 页 / 共 Y 页」，是活的域不是死字 +``` + +目录是**域**，内容由 Word 打开时生成。`add_toc` 已经写了 `updateFields`， +用户打开会自动刷新；不写这个标记的话目录永远显示占位文字，看起来像坏了。 + +## 4. 表格 + +```python +add_table( + doc, + headers=["项目", "2024年", "2025E", "同比"], + rows=[["营业收入", "12,500", "14,200", "13.6%"], + ["营业成本", " 8,200", " 9,100", "11.0%"]], + widths_cm=[5, 3.6, 3.6, 3.2], # ★ 合计不要超过版心 + style="Table Grid", +) +print("版心宽度 cm:", content_width_cm(section)) +``` + +> **列宽必须写到每个单元格**。只设 `table.columns[i].width` Word 会忽略 —— +> 这是"表格被压扁/撑出页面"的头号原因。`add_table` 已经处理了，手写时要自己循环设。 +> 另外 `table.autofit` 必须为 False，否则宽度还是会被重算。 + +跨页的长表要重复表头，靠首行的 `w:tblHeader`（`add_table` 已自动设）： + +```python +from docx_helpers import repeat_header_row +repeat_header_row(table) # 手写表格补这一句，否则第二页只剩一堆数字 +``` + +合并单元格： + +```python +a = table.cell(0, 0); b = table.cell(0, 1) +a.merge(b) +``` + +单元格底纹用 `set_cell_shading(cell, "F2F6FA")`（python-docx 没有 API，要写 XML）。 + +## 5. 分隔线 + +```python +add_hr(doc) # 段落下边框 +``` + +> **不要用单行表格当分隔线** —— 会打断文字流，屏幕阅读器也读不对。 + +## 6. 图片 + +```python +add_image_fitted(doc, "scratch/trend.png", section=section, caption="图 1 分季度收入趋势") +``` + +它按版心宽度封顶等比缩放（只缩不放，小图保持原尺寸），避免图片撑出页面（体检会抓超宽图）。 +图表先用 matplotlib 画成 PNG 存 `scratch/`，再插进来。 + +> matplotlib 画中文要先设字体，否则全是方框： +> ```python +> import matplotlib +> matplotlib.rc("font", family=["WenQuanYi Zen Hei", "Noto Sans CJK SC", "sans-serif"]) +> matplotlib.rcParams["axes.unicode_minus"] = False +> ``` +> 写成列表是因为不同机器装的中文字体不一样（镜像是文泉驿，有的机器只有 Noto）。 + +## 7. 列表 + +```python +doc.add_paragraph("第一项", style="List Bullet") +doc.add_paragraph("第一步", style="List Number") +``` + +> **不要手写 `•` 或 `1.`** —— 那是假列表，缩进和续行都不对。 +> 写完记得 `set_run_font` 补中文字体（`add_paragraph(text, style=...)` 生成的 run 也要设）。 + +## 8. 分页与分节 + +```python +doc.add_page_break() + +from docx.enum.section import WD_SECTION +new_section = doc.add_section(WD_SECTION.NEW_PAGE) # 想换成横向就在新节里改 +new_section.page_width, new_section.page_height = section.page_height, section.page_width +``` + +## 9. 页眉 + +```python +header = section.header +p = header.paragraphs[0] +set_run_font(p.add_run("XX 公司 · 内部资料"), CN_FONT, 9, color="808080") +p.alignment = WD_ALIGN_PARAGRAPH.RIGHT +``` + +## 10. 读已有文档 + +```python +doc = Document("uploads/原文.docx") +for p in doc.paragraphs: + print(p.style.name, "|", p.text) +for t in doc.tables: + for row in t.rows: + print([c.text for c in row.cells]) +``` + +改已有文档时：**只改需要改的 run**，不要整段重建 —— 重建会丢掉原有的字体、编号、批注关联。 + +```python +for p in doc.paragraphs: + for r in p.runs: + if "旧词" in r.text: + r.text = r.text.replace("旧词", "新词") # 保留该 run 的全部格式 +``` + +> Word 会把一句话拆进很多个 `w:r`（拼写检查、修订标记都会拆）， +> 所以你**看得见**的短语在 XML 里往往不是连续字符串。跨 run 的替换要先合并 run， +> 或退一步用整段重写 + 手动补格式。 + +## 11. 保存 + +```python +doc.save("output/2024年度经营分析报告.docx") +print("saved") +``` diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/docx_helpers.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/docx_helpers.py new file mode 100644 index 0000000000..a8fa7e0bbe --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/docx_helpers.py @@ -0,0 +1,341 @@ +"""python-docx helpers for building .docx inside the BiSheng code interpreter. + +Use from a build script: + + import sys + sys.path.insert(0, "skills/bisheng-docx/scripts") + from docx_helpers import set_run_font, apply_chinese_defaults, add_table + +Every function here exists because plain python-docx gets the Chinese case +wrong or cannot express the construct at all: + +* ``run.font.name`` writes only ``w:ascii``/``w:hAnsi`` — Chinese glyphs keep + falling back to the theme font. The CJK face lives in ``w:eastAsia`` and has + to be set through the XML. +* Table column widths must be written to **every cell**; setting them on the + column object alone is silently ignored by Word. +* A table of contents, page numbers, and a horizontal rule are all field codes + or borders with no python-docx API at all. +""" + +from docx.enum.table import WD_TABLE_ALIGNMENT +from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.oxml import OxmlElement +from docx.oxml.ns import qn +from docx.shared import Cm, Pt, RGBColor + +# Fonts a Chinese user's Word certainly has. Rendering happens on their machine, +# so an exotic face here becomes a substitution there. +CN_FONT = "微软雅黑" +CN_FONT_SERIF = "宋体" +CN_FONT_HEADING = "微软雅黑" +LATIN_FONT = "Times New Roman" + +HEADER_FILL = "1F4E79" +BAND_FILL = "F2F6FA" + + +def set_run_font(run, name: str | None = None, size_pt: float | None = None, bold=None, color: str | None = None): + """Set a run's font so it applies to Chinese too. + + ``run.font.name`` only writes ``w:ascii`` and ``w:hAnsi``; without a + matching ``w:eastAsia`` entry Word renders CJK in the theme font and the + document silently ignores the face you asked for. + """ + if name: + run.font.name = name + rpr = run._element.get_or_add_rPr() + rfonts = rpr.find(qn("w:rFonts")) + if rfonts is None: + rfonts = OxmlElement("w:rFonts") + rpr.insert(0, rfonts) + rfonts.set(qn("w:ascii"), name) + rfonts.set(qn("w:hAnsi"), name) + rfonts.set(qn("w:eastAsia"), name) + if size_pt is not None: + run.font.size = Pt(size_pt) + if bold is not None: + run.font.bold = bold + if color: + run.font.color.rgb = RGBColor.from_string(color) + return run + + +def _style_font(style, cn_name: str, size_pt: float | None = None): + style.font.name = cn_name + rpr = style.element.get_or_add_rPr() + rfonts = rpr.find(qn("w:rFonts")) + if rfonts is None: + rfonts = OxmlElement("w:rFonts") + rpr.insert(0, rfonts) + rfonts.set(qn("w:ascii"), cn_name) + rfonts.set(qn("w:hAnsi"), cn_name) + rfonts.set(qn("w:eastAsia"), cn_name) + if size_pt is not None: + style.font.size = Pt(size_pt) + + +def apply_chinese_defaults(doc, body_font: str = CN_FONT, heading_font: str = CN_FONT_HEADING, body_pt: float = 11): + """Make Normal and Heading 1–4 render correctly in Chinese. + + Word's stock Heading styles are Calibri Light in a blue nobody asked for; + left alone they make every generated document look identically foreign. + """ + _style_font(doc.styles["Normal"], body_font, body_pt) + doc.styles["Normal"].paragraph_format.line_spacing = 1.5 + doc.styles["Normal"].paragraph_format.space_after = Pt(6) + + sizes = {1: 20, 2: 16, 3: 14, 4: 12} + for level, size in sizes.items(): + try: + style = doc.styles[f"Heading {level}"] + except KeyError: + continue + _style_font(style, heading_font, size) + style.font.bold = True + # Word's stock Heading 4 (and several template variants) are italic. + # Left alone, a level-4 Chinese heading renders as slanted serif and + # looks like a different document. + style.font.italic = False + style.font.color.rgb = RGBColor.from_string("1F1F1F") + style.paragraph_format.space_before = Pt(12 if level > 1 else 18) + style.paragraph_format.space_after = Pt(6) + + +def setup_page(doc, width_cm: float = 21.0, height_cm: float = 29.7, margin_cm: float = 2.54, landscape: bool = False): + """A4 portrait by default; pass landscape=True to swap the dimensions.""" + for section in doc.sections: + section.page_width = Cm(height_cm if landscape else width_cm) + section.page_height = Cm(width_cm if landscape else height_cm) + section.left_margin = section.right_margin = Cm(margin_cm) + section.top_margin = section.bottom_margin = Cm(margin_cm) + return doc.sections[0] + + +def content_width_cm(section) -> float: + """Usable width between the margins, in cm — the cap for tables and images.""" + return (section.page_width - section.left_margin - section.right_margin) / 360000 + + +def _field(paragraph, instruction: str, placeholder: str = ""): + """Insert a Word field code. Fields are how TOC and page numbers work.""" + run = paragraph.add_run() + begin = OxmlElement("w:fldChar") + begin.set(qn("w:fldCharType"), "begin") + run._element.append(begin) + + run = paragraph.add_run() + instr = OxmlElement("w:instrText") + instr.set(qn("xml:space"), "preserve") + instr.text = instruction + run._element.append(instr) + + run = paragraph.add_run() + sep = OxmlElement("w:fldChar") + sep.set(qn("w:fldCharType"), "separate") + run._element.append(sep) + + if placeholder: + paragraph.add_run(placeholder) + + run = paragraph.add_run() + end = OxmlElement("w:fldChar") + end.set(qn("w:fldCharType"), "end") + run._element.append(end) + return paragraph + + +def enable_update_fields(doc): + """Ask Word to refresh every field when the document is opened. + + Without this the TOC shows the placeholder text until someone presses F9, + which reads as a broken document. + """ + settings = doc.settings.element + existing = settings.find(qn("w:updateFields")) + if existing is None: + existing = OxmlElement("w:updateFields") + settings.append(existing) + existing.set(qn("w:val"), "true") + + +def add_toc(doc, levels: str = "1-3", title: str = "目录"): + """Insert a table of contents field. Requires built-in Heading styles.""" + if title: + heading = doc.add_paragraph() + run = heading.add_run(title) + set_run_font(run, CN_FONT_HEADING, 18, bold=True) + heading.alignment = WD_ALIGN_PARAGRAPH.CENTER + paragraph = doc.add_paragraph() + _field(paragraph, f' TOC \\o "{levels}" \\h \\z \\u ', "（打开文档时自动生成；若未显示请按 Ctrl+A 后 F9）") + enable_update_fields(doc) + return paragraph + + +def add_page_number_footer(section, template: str = "第 {PAGE} 页 / 共 {NUMPAGES} 页", size_pt: float = 9): + """Page numbers in the footer, as live fields rather than baked-in text.""" + footer = section.footer + paragraph = footer.paragraphs[0] if footer.paragraphs else footer.add_paragraph() + paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER + for chunk in _split_template(template): + if chunk in ("PAGE", "NUMPAGES"): + _field(paragraph, f" {chunk} ", "1") + else: + paragraph.add_run(chunk) + for run in paragraph.runs: + set_run_font(run, CN_FONT, size_pt, color="808080") + return paragraph + + +def _split_template(template: str): + parts, buffer, index = [], "", 0 + while index < len(template): + if template[index] == "{": + close = template.find("}", index) + if close > 0: + if buffer: + parts.append(buffer) + buffer = "" + parts.append(template[index + 1 : close]) + index = close + 1 + continue + buffer += template[index] + index += 1 + if buffer: + parts.append(buffer) + return parts + + +def set_cell_shading(cell, hex_fill: str): + tc_pr = cell._tc.get_or_add_tcPr() + shd = OxmlElement("w:shd") + shd.set(qn("w:val"), "clear") + shd.set(qn("w:color"), "auto") + shd.set(qn("w:fill"), hex_fill) + tc_pr.append(shd) + + +def add_table(doc, headers, rows, widths_cm=None, style: str = "Table Grid", font_pt: float = 10, banded=True): + """Add a table with widths Word will actually honour. + + python-docx exposes ``column.width``, but Word ignores it unless the same + width is written to every cell in that column and autofit is off — the + single most common "why is my table squashed" bug. + """ + table = doc.add_table(rows=1, cols=len(headers)) + if style: + try: + table.style = style + except KeyError: + pass # style missing from the template; grid lines are cosmetic + table.alignment = WD_TABLE_ALIGNMENT.CENTER + table.autofit = widths_cm is None + + header_cells = table.rows[0].cells + for index, title in enumerate(headers): + cell = header_cells[index] + cell.text = "" + run = cell.paragraphs[0].add_run(str(title)) + set_run_font(run, CN_FONT, font_pt, bold=True, color="FFFFFF") + cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.CENTER + set_cell_shading(cell, HEADER_FILL) + + for row_index, record in enumerate(rows): + cells = table.add_row().cells + for index, value in enumerate(record): + cell = cells[index] + cell.text = "" + run = cell.paragraphs[0].add_run("" if value is None else str(value)) + set_run_font(run, CN_FONT, font_pt) + if isinstance(value, (int, float)): + cell.paragraphs[0].alignment = WD_ALIGN_PARAGRAPH.RIGHT + if banded and row_index % 2 == 1: + set_cell_shading(cell, BAND_FILL) + + if widths_cm: + for row in table.rows: + for index, width in enumerate(widths_cm): + if index < len(row.cells): + row.cells[index].width = Cm(width) + repeat_header_row(table) + return table + + +def repeat_header_row(table): + """Repeat row 0 on every page the table spills onto. + + Word only does this when the row carries ``w:tblHeader``; without it a long + table's second page is a wall of numbers with no column names. + """ + if not table.rows: + return table + tr_pr = table.rows[0]._tr.get_or_add_trPr() + if tr_pr.find(qn("w:tblHeader")) is None: + tr_pr.append(OxmlElement("w:tblHeader")) + return table + + +def add_hr(doc, color: str = "BFBFBF", size: int = 6): + """A horizontal rule as a paragraph bottom border. + + Never use a one-row table for this — it breaks text flow and screen readers. + """ + paragraph = doc.add_paragraph() + p_pr = paragraph._p.get_or_add_pPr() + borders = OxmlElement("w:pBdr") + bottom = OxmlElement("w:bottom") + bottom.set(qn("w:val"), "single") + bottom.set(qn("w:sz"), str(size)) + bottom.set(qn("w:space"), "1") + bottom.set(qn("w:color"), color) + borders.append(bottom) + p_pr.append(borders) + return paragraph + + +def add_image_fitted(doc, image_path: str, section=None, max_width_cm: float | None = None, caption: str | None = None): + """Insert an image capped at the text width, optionally captioned. + + Only ever shrinks: forcing every picture to the full text width blows a + 600px chart up to 16cm and it renders visibly soft. + """ + if max_width_cm is None: + max_width_cm = content_width_cm(section or doc.sections[0]) + paragraph = doc.add_paragraph() + paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER + picture = paragraph.add_run().add_picture(image_path) + limit = Cm(max_width_cm) + if picture.width > limit: + picture.height = int(picture.height * limit / picture.width) + picture.width = limit + if caption: + cap = doc.add_paragraph() + cap.alignment = WD_ALIGN_PARAGRAPH.CENTER + set_run_font(cap.add_run(caption), CN_FONT, 9, color="808080") + return paragraph + + +def first_line_indent(paragraph, chars: float = 2, font_pt: float = 11): + """Chinese body text is indented by two characters, not by a tab.""" + paragraph.paragraph_format.first_line_indent = Pt(chars * font_pt) + return paragraph + + +def add_body(doc, text: str, indent: bool = True, font_pt: float = 11): + """A body paragraph with the Chinese conventions already applied.""" + paragraph = doc.add_paragraph() + set_run_font(paragraph.add_run(text), CN_FONT, font_pt) + if indent: + first_line_indent(paragraph, 2, font_pt) + return paragraph + + +def add_heading_cn(doc, text: str, level: int = 1): + """A heading that keeps the built-in style (so the TOC finds it) but renders + in a Chinese face.""" + heading = doc.add_heading(level=level) + run = set_run_font( + heading.add_run(text), CN_FONT_HEADING, {1: 20, 2: 16, 3: 14, 4: 12}.get(level, 12), bold=True, color="1F1F1F" + ) + run.font.italic = False # stock Heading 4 is italic; override at run level too + return heading diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/inspect_docx.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/inspect_docx.py new file mode 100644 index 0000000000..5567757082 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/inspect_docx.py @@ -0,0 +1,477 @@ +#!/usr/bin/env python3 +"""Read out and health-check a .docx, for the BiSheng code interpreter. + + import subprocess, sys + r = subprocess.run([sys.executable, "skills/bisheng-docx/scripts/inspect_docx.py", + "output/x.docx"], capture_output=True, text=True) + print(r.stdout or "(no stdout)") + print(r.stderr[-2000:] if r.stderr else "(no stderr)") + +Flags: ``--content-only`` (dump only), ``--check-only`` (checks only), +``--max-paras N``. + +Replaces two things the official skill relies on and this environment lacks: +``pandoc -t markdown`` for reading (pandoc exists in the release image but not +on every hand-built host, and it drops the structural detail needed to plan an +edit), and "render to JPG and look at every page", which assumes a model that +can see images. + +The single highest-value check is the missing ``w:eastAsia`` font: it is +invisible in python-docx, invisible in extracted text, and shows up only when +the user opens the file and finds every Chinese character in the wrong face. +The check walks the style inheritance chain, so a run that legitimately +inherits a CJK face from ``Normal`` is not reported. + +Always exits 0 — the executor discards stdout on a non-zero exit. +""" + +import argparse +import os +import sys +import traceback + +try: + from docx import Document + from docx.oxml.ns import qn +except ImportError: # pragma: no cover + print("[FATAL] python-docx 不可用；无法体检。") + sys.exit(0) + +PLACEHOLDERS = ["待填", "待补", "TODO", "XXX", "占位", "请填写", "TBD", "lorem ipsum", "示例文字"] +CJK_RANGE = ("一", "鿿") +MIN_RUN_PT = 9.0 # design-zh: 图注 9pt 是最小合法字号 +MIN_BODY_PT = 10.5 # design-zh: 正文 11pt +REPEAT_HEADER_ROWS = 12 # 超过这个行数的表跨页时必须重复表头 + +findings: list[tuple[str, str]] = [] + + +def add(level: str, message: str) -> None: + findings.append((level, message)) + + +def has_cjk(text: str) -> bool: + return any(CJK_RANGE[0] <= ch <= CJK_RANGE[1] for ch in text) + + +def _east_asia_of(rpr) -> str | None: + if rpr is None: + return None + rfonts = rpr.find(qn("w:rFonts")) + if rfonts is None: + return None + return rfonts.get(qn("w:eastAsia")) + + +def run_east_asia(run) -> str | None: + """The w:eastAsia face written on this run itself, if any.""" + return _east_asia_of(run._element.find(qn("w:rPr"))) + + +def style_east_asia(style) -> str | None: + """The w:eastAsia face written on this style itself (no inheritance).""" + if style is None: + return None + return _east_asia_of(style.element.find(qn("w:rPr"))) + + +def inherited_east_asia(style) -> str | None: + """Walk basedOn up the chain: a run under Heading 1 inherits Normal's face.""" + depth = 0 + while style is not None and depth < 8: + face = style_east_asia(style) + if face: + return face + try: + style = style.base_style + except (KeyError, AttributeError): + return None + depth += 1 + return None + + +def style_size_pt(style) -> float | None: + try: + size = style.font.size + except (KeyError, AttributeError): + return None + return None if size is None else size.pt + + +def iter_block_paragraphs(doc): + """Body paragraphs plus every paragraph inside a table cell.""" + for paragraph in doc.paragraphs: + yield paragraph, None + for t_index, table in enumerate(doc.tables): + for r_index, row in enumerate(table.rows): + for c_index, cell in enumerate(row.cells): + for paragraph in cell.paragraphs: + yield paragraph, f"表{t_index + 1}[{r_index + 1},{c_index + 1}]" + + +def heading_levels(doc) -> list[tuple[int, str]]: + levels: list[tuple[int, str]] = [] + for paragraph in doc.paragraphs: + name = paragraph.style.name if paragraph.style is not None else "" + if name.startswith("Heading ") and paragraph.text.strip(): + try: + levels.append((int(name.split()[-1]), paragraph.text.strip())) + except ValueError: + continue + return levels + + +def header_footer_xml(doc) -> str: + """Raw XML of every header/footer part — PAGE fields do NOT live in document.xml.""" + chunks = [] + for part in doc.part.package.iter_parts(): + name = str(getattr(part, "partname", "")) + if "header" in name or "footer" in name: + try: + chunks.append(part.blob.decode("utf-8", "ignore")) + except Exception: + continue + return "\n".join(chunks) + + +def field_instructions(doc) -> list[str]: + return [element.text or "" for element in doc.element.iter(qn("w:instrText"))] + + +# --------------------------------------------------------------------------- # +# Content read-out +# --------------------------------------------------------------------------- # + + +def dump_content(path: str, max_paras: int) -> None: + doc = Document(path) + section = doc.sections[0] + width_cm = section.page_width / 360000 + height_cm = section.page_height / 360000 + + print("=== 内容 ===") + print( + f"{os.path.basename(path)} 页面 {width_cm:.1f}×{height_cm:.1f}cm " + f"页边距 上{section.top_margin / 360000:.1f} 下{section.bottom_margin / 360000:.1f} " + f"左{section.left_margin / 360000:.1f} 右{section.right_margin / 360000:.1f}cm " + f"段落 {len(doc.paragraphs)} 表格 {len(doc.tables)} 图片 {len(doc.inline_shapes)}" + ) + print() + + shown = 0 + for index, paragraph in enumerate(doc.paragraphs): + text = paragraph.text.strip() + if not text: + continue + if shown >= max_paras: + print(" … 其余段落未显示（--max-paras 可调）") + break + style = paragraph.style.name if paragraph.style is not None else "?" + tag = f"[{style}]" if style != "Normal" else "" + print(f"P{index:<4} {tag} {text if len(text) <= 110 else text[:107] + '…'}") + shown += 1 + + for t_index, table in enumerate(doc.tables): + print(f"\n## 表 {t_index + 1} · {len(table.rows)} 行 × {len(table.columns)} 列") + for row in table.rows[:8]: + cells = [c.text.strip().replace("\n", " ")[:24] for c in row.cells] + print(" | " + " | ".join(cells) + " |") + if len(table.rows) > 8: + print(f" … 其余 {len(table.rows) - 8} 行未显示") + + +# --------------------------------------------------------------------------- # +# Health checks +# --------------------------------------------------------------------------- # + + +def check_fonts(doc) -> None: + """The headline check: every Chinese run must end up with a w:eastAsia face.""" + if not style_east_asia(doc.styles["Normal"]): + add( + "WARN", + "Normal 样式没有设置 w:eastAsia 中文字体 —— 中文会退回主题字体（通常是等线/宋体），" + "和你指定的西文字体对不上。在构建脚本里调 apply_chinese_defaults(doc) 一次性设好。", + ) + + uncovered: list[str] = [] # 完全没有中文字体可继承 + mismatched: dict[str, str] = {} # run 写了西文字体、中文却继承自样式 + style_gaps: dict[str, str] = {} # 该段落样式整条继承链都没有中文字体 + checked = 0 + for paragraph, where in iter_block_paragraphs(doc): + style = paragraph.style + style_face = inherited_east_asia(style) + for run in paragraph.runs: + if not has_cjk(run.text): + continue + checked += 1 + label = where or f"「{run.text.strip()[:16]}」" + east = run_east_asia(run) + latin = run.font.name + if east: + continue + if not style_face: + if latin and label not in uncovered: + uncovered.append(label) + elif not latin: + style_gaps.setdefault(style.name if style is not None else "?", label) + elif latin and latin != style_face: + mismatched.setdefault(label, f"{latin}→实际 {style_face}") + + if uncovered: + add( + "ERROR", + f"{len(uncovered)} 处中文 run 只设了西文字体、没设 w:eastAsia，且样式里也没有中文字体：" + f"{', '.join(uncovered[:6])}{' …' if len(uncovered) > 6 else ''}。" + "`run.font.name = '微软雅黑'` 对中文无效 —— 改用 set_run_font(run, '微软雅黑', size_pt=11)。", + ) + if style_gaps: + names = ", ".join(f"{k}（如{v}）" for k, v in list(style_gaps.items())[:4]) + add( + "ERROR", + f"{len(style_gaps)} 个段落样式的继承链里没有 w:eastAsia 中文字体：{names}。" + "这些中文会退回主题字体，用户打开就是「字体乱了」。" + "调 apply_chinese_defaults(doc)（它会把 Normal 和 Heading 1–4 一次设好），" + "或用 add_heading_cn / add_body 写内容。", + ) + if mismatched: + items = ", ".join(f"{k}:{v}" for k, v in list(mismatched.items())[:4]) + add( + "WARN", + f"{len(mismatched)} 处中文 run 写的是西文字体、中文实际用的是样式继承来的另一种字体：{items}。" + "同一段中西文字体不一致会看出明显断层 —— 用 set_run_font() 把三个字段一起设。", + ) + if checked == 0: + add("INFO", "文档里没有中文 run（纯英文文档？）") + + small: list[str] = [] + for paragraph, where in iter_block_paragraphs(doc): + for run in paragraph.runs: + if not run.text.strip(): + continue + size = run.font.size + if size is not None and size.pt < MIN_RUN_PT: + small.append(f"{where or run.text.strip()[:12]}={size.pt:g}pt") + if small: + add( + "WARN", + f"{len(small)} 处文字字号小于 {MIN_RUN_PT:g}pt：{', '.join(small[:6])}" + f"{' …' if len(small) > 6 else ''}。打印后基本读不清 —— " + f"正文 11pt、表格 10pt、图注 9pt，最小不要低于 {MIN_RUN_PT:g}pt。", + ) + normal_pt = style_size_pt(doc.styles["Normal"]) + if normal_pt is not None and normal_pt < MIN_BODY_PT: + add( + "WARN", + f"Normal 样式字号 {normal_pt:g}pt 偏小（正文规范 11pt）。" + "在 apply_chinese_defaults(doc, body_pt=11) 里调回来。", + ) + + +def check_headings(doc) -> None: + levels = heading_levels(doc) + if not levels: + add( + "WARN", + "文档里没有用内置 Heading 样式的标题 —— 目录生成不出来，导航窗格也是空的。" + "用 add_heading_cn(doc, '一、xxx', 2)，不要用加粗的普通段落冒充标题。", + ) + return + previous = 0 + for level, text in levels: + if previous and level > previous + 1: + add( + "WARN", + f"标题层级从 H{previous} 直接跳到 H{level}：「{text[:24]}」—— 中间补一级，或把它降到 H{previous + 1}", + ) + previous = level + + +def _cell_shaded(cell) -> bool: + tc_pr = cell._tc.tcPr + if tc_pr is None: + return False + shd = tc_pr.find(qn("w:shd")) + if shd is None: + return False + fill = (shd.get(qn("w:fill")) or "auto").lower() + return fill not in ("auto", "ffffff", "") + + +def _row_has_bold(row) -> bool: + for cell in row.cells: + for paragraph in cell.paragraphs: + for run in paragraph.runs: + if run.font.bold or (run.style is not None and getattr(run.style.font, "bold", None)): + return True + return False + + +def _row_repeats_as_header(row) -> bool: + tr_pr = row._tr.find(qn("w:trPr")) + return tr_pr is not None and tr_pr.find(qn("w:tblHeader")) is not None + + +def check_tables(doc, section) -> None: + usable_cm = (section.page_width - section.left_margin - section.right_margin) / 360000 + for index, table in enumerate(doc.tables): + if not table.rows: + add("WARN", f"表 {index + 1} 是空的 —— 删掉它，或把数据填进去") + continue + widths = [c.width for c in table.rows[0].cells] + if all(w is not None for w in widths): + total_cm = sum(w / 360000 for w in widths) + if total_cm > usable_cm + 0.2: + add( + "ERROR", + f"表 {index + 1} 总宽 {total_cm:.1f}cm 超过版心 {usable_cm:.1f}cm → 右侧列会被截出页面。" + f"把 widths_cm 的合计压到 {usable_cm:.1f}cm 以内（用 content_width_cm(section) 取准确值）。", + ) + elif any(w is None for w in widths): + add( + "INFO", + f"表 {index + 1} 没有设列宽，Word 会自动分配；列多时中文列容易被压成竖排。" + f"给 add_table 传 widths_cm=[…]（合计 ≤ {usable_cm:.1f}cm）。", + ) + header = table.rows[0] + if not _cell_shaded(header.cells[0]) and not _row_has_bold(header): + add( + "WARN", + f"表 {index + 1} 首行不像表头（既没有加粗也没有底纹）—— 读者分不清表头和数据。" + "用 add_table(doc, headers=[…], rows=[…])，它会自动加深底白字加粗。", + ) + if len(table.rows) > REPEAT_HEADER_ROWS and not _row_repeats_as_header(header): + add( + "INFO", + f"表 {index + 1} 有 {len(table.rows)} 行，跨页后第二页没有表头。" + "add_table 已自动设置重复表头；手写表格用 repeat_header_row(table) 补上。", + ) + for r_index, row in enumerate(table.rows): + if len(row.cells) != len(table.columns): + add( + "WARN", + f"表 {index + 1} 第 {r_index + 1} 行单元格数 {len(row.cells)} 与列数 {len(table.columns)} 不一致" + "（可能有合并单元格）—— 确认是有意为之，否则补齐这一行。", + ) + break + + +def check_images(doc, section) -> None: + usable_cm = (section.page_width - section.left_margin - section.right_margin) / 360000 + for index, shape in enumerate(doc.inline_shapes): + width_cm = shape.width / 360000 + if width_cm > usable_cm + 0.2: + add( + "ERROR", + f"图 {index + 1} 宽 {width_cm:.1f}cm 超过版心 {usable_cm:.1f}cm → 会溢出页面。" + "用 add_image_fitted(doc, path, section=section) 按版心等比缩放。", + ) + + +def check_text(doc) -> None: + empty_streak = 0 + for paragraph in doc.paragraphs: + text = paragraph.text.strip() + if not text: + empty_streak += 1 + if empty_streak == 3: + add("INFO", "有连续 3 个以上空段落 —— 改用段前段后间距（space_before/space_after）分隔内容") + else: + empty_streak = 0 + + for paragraph, where in iter_block_paragraphs(doc): + text = paragraph.text.strip() + if not text: + continue + prefix = f"{where} " if where else "" + for token in PLACEHOLDERS: + if token in text: + add("WARN", f"{prefix}残留占位文本「{token}」：{text[:40]} —— 换成真实内容，或整段删掉") + break + if "\n" in paragraph.text: + add( + "WARN", f"{prefix}段落里有换行符而不是独立段落：{text[:40]} —— 每段一个 Paragraph，或用 run.add_break()" + ) + + +def check_fields(doc) -> None: + instructions = field_instructions(doc) + hdr_ftr = header_footer_xml(doc) + has_toc = any("TOC" in text for text in instructions) + if has_toc: + if "updateFields" not in doc.settings.element.xml: + add( + "ERROR", + "有目录域但没开 updateFields —— 用户打开时目录仍是占位文字，看起来像坏了。" + "调用 enable_update_fields(doc)（add_toc 已内置）。", + ) + if not heading_levels(doc): + add( + "ERROR", + "有目录域但全文没有内置 Heading 样式的标题 → 目录会是空的。" + "标题一律用 add_heading_cn(doc, text, level)。", + ) + has_page_field = any("PAGE" in text for text in instructions) or "PAGE" in hdr_ftr + if not has_page_field and len(doc.paragraphs) > 40: + add( + "INFO", + f"文档有 {len(doc.paragraphs)} 个段落但页脚没有页码域 —— 调 add_page_number_footer(section)。", + ) + + +def run_checks(path: str) -> None: + doc = Document(path) + section = doc.sections[0] + check_fonts(doc) + check_headings(doc) + check_tables(doc, section) + check_images(doc, section) + check_text(doc) + check_fields(doc) + + print() + print("=== 体检 ===") + + order = {"ERROR": 0, "WARN": 1, "INFO": 2} + counts = {"ERROR": 0, "WARN": 0, "INFO": 0} + for level, _ in findings: + counts[level] += 1 + for level, message in sorted(findings, key=lambda f: order[f[0]]): + print(f"[{level}] {message}") + if not findings: + print("（无发现）") + + print() + print(f"合计: {counts['ERROR']} ERROR / {counts['WARN']} WARN / {counts['INFO']} INFO") + if counts["ERROR"]: + print(f"结论: 不通过 —— {counts['ERROR']} 项 ERROR 必须修完。改构建脚本重新生成，再跑一次本脚本。") + else: + print("结论: 通过 —— 无必须修复项。WARN 逐条复核后即可交付。") + + +def main() -> None: + parser = argparse.ArgumentParser(description="读出并体检一个 .docx") + parser.add_argument("path") + parser.add_argument("--content-only", action="store_true") + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--max-paras", type=int, default=80) + args = parser.parse_args() + + if not os.path.exists(args.path): + print(f"[FATAL] 文件不存在: {args.path}") + print("提示：代码执行器 cwd 就是工作区根，用相对路径 `output/x.docx`，不要写 `/output/x.docx`。") + return + + try: + if not args.check_only: + dump_content(args.path, args.max_paras) + if not args.content_only: + run_checks(args.path) + except Exception: + print("[FATAL] 体检过程本身出错：") + traceback.print_exc(file=sys.stdout) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/render_docx.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/render_docx.py new file mode 100644 index 0000000000..83d9fcdde2 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-docx/scripts/render_docx.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Render a .docx to per-page PNGs so the layout can be eyeballed. + + import subprocess, sys + r = subprocess.run([sys.executable, "skills/bisheng-docx/scripts/render_docx.py", + "output/x.docx"], capture_output=True, text=True) + +Flags: ``--dpi 110``, ``--timeout 180``, ``--outdir``. + +The official skill does this with ``soffice`` + ``pdftoppm``. Poppler is not in +the BiSheng image, so the PDF is rasterised with PyMuPDF instead — same result, +one fewer missing binary, and it keeps the ``page-01.png`` zero-padded naming +the rest of the workflow expects. + +.docx goes through LibreOffice **Writer** (present in the release image and on +114); only .pptx needs Impress. Optional step either way: it only helps when the +model can actually read images back, and a PNG sent to a model that cannot read +images can fail the whole request. When LibreOffice is absent or cannot load the +file, the script says so plainly and ``inspect_docx.py`` remains the source of +truth. + +Always exits 0 — the executor discards stdout on a non-zero exit. +""" + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +import traceback + + +def find_soffice() -> str | None: + return shutil.which("soffice") or shutil.which("libreoffice") + + +def to_pdf(src: str, outdir: str, timeout: int) -> str | None: + soffice = find_soffice() + if not soffice: + print("[跳过] 环境里没有 LibreOffice（soffice），无法渲染预览。") + print(" 这不影响 .docx 的生成与交付 —— 以 inspect_docx.py 的体检结果为准。") + return None + + os.makedirs(outdir, exist_ok=True) + # A throwaway profile per run, deleted afterwards: a fixed profile path is + # what makes concurrent conversions in a multi-task worker corrupt each + # other, and a leaked one leaves ~0.5MB of junk in /tmp per render. + with tempfile.TemporaryDirectory(prefix="lo_profile_") as profile: + cmd = [ + soffice, + "--headless", + "--norestore", + "--invisible", + "-env:SingleAppInstance=false", + f"-env:UserInstallation=file://{profile}", + "--convert-to", + "pdf", + "--outdir", + outdir, + os.path.abspath(src), + ] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + print(f"[失败] LibreOffice 转换超时（超过 {timeout}s），没有产出预览图。") + print(" 这只影响可选的视觉检查 —— 以 inspect_docx.py 的体检结果为准，不要因此重做文档。") + return None + + # soffice exits 0 even when it converted nothing → 只认产出文件，不认退出码。 + pdf = os.path.join(outdir, os.path.splitext(os.path.basename(src))[0] + ".pdf") + if not os.path.exists(pdf): + message = ((proc.stdout or "") + (proc.stderr or "")).strip() + print(f"[跳过] LibreOffice 没能转出 PDF（退出码 {proc.returncode}）：{message[:200]}") + if "source file could not be loaded" in message.lower(): + print(" 这台机器的 LibreOffice 很可能没装 Writer 组件，请让运维补装 libreoffice-writer。") + elif proc.returncode == 137: + print(" 退出码 137 = 被 OOM killer 杀掉，是内存配额太小，不是 LibreOffice 坏了。") + return None + return pdf + + +def rasterise(pdf: str, outdir: str, dpi: int) -> list[str]: + try: + import fitz + except ImportError: + print("[跳过] PyMuPDF(fitz) 不可用，无法把 PDF 转成图片。") + return [] + + doc = fitz.open(pdf) + pad = max(2, len(str(doc.page_count))) + written = [] + for index, page in enumerate(doc, start=1): + pixmap = page.get_pixmap(dpi=dpi) + path = os.path.join(outdir, f"page-{index:0{pad}d}.png") + pixmap.save(path) + written.append(path) + doc.close() + return written + + +def main() -> None: + parser = argparse.ArgumentParser(description="把 .docx 渲染成逐页 PNG") + parser.add_argument("path") + parser.add_argument("--dpi", type=int, default=110) + parser.add_argument("--timeout", type=int, default=180) + parser.add_argument("--outdir", default=None, help="默认 scratch/preview/<文件名>/") + args = parser.parse_args() + + if not os.path.exists(args.path): + print(f"[FATAL] 文件不存在: {args.path}") + print("提示：用相对路径 `output/x.docx`，不要写 `/output/x.docx`。") + return + + stem = os.path.splitext(os.path.basename(args.path))[0] + outdir = args.outdir or os.path.join("scratch", "preview", stem) + + try: + os.makedirs(outdir, exist_ok=True) + pdf = to_pdf(args.path, outdir, args.timeout) + if not pdf: + return + pages = rasterise(pdf, outdir, args.dpi) + if not pages: + return + print(f"已渲染 {len(pages)} 页到 {outdir}/") + for path in pages: + print(f" {path}") + print() + print("用 read_file 逐张查看这些 PNG。注意：服务端只有文泉驿正黑一种中文字体，") + print("和用户 Word 里的实际字体宽度不同 —— 预览里的行长松紧只作参考，") + print("不要为了预览效果反复微调字号，以 inspect_docx.py 的体检结果为准。") + except Exception: + print("[FATAL] 渲染过程本身出错：") + traceback.print_exc(file=sys.stdout) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/SKILL.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/SKILL.md new file mode 100644 index 0000000000..67a05d4b3a --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/SKILL.md @@ -0,0 +1,204 @@ +--- +name: bisheng-pptx +description: >- + 制作 PowerPoint 演示文稿（.pptx）时使用：从零创建企业介绍、工作汇报、项目方案、产品发布、培训课件等幻灯片； + 按用户提供的模板（.pptx/.potx）套版生成；读取或改写已有 PPT。当用户说到「PPT」「幻灯片」「演示文稿」「汇报材料」 + 「宣讲材料」「课件」「deck」「slides」「pptx」，或提出「做一个介绍 X 的 PPT」「按这个模板做一版」 + 「把这份材料做成 PPT」时触发。本技能给出 BiSheng 代码执行器里可行的 python-pptx 生成路径， + 以及中文排版规范、模板套用方法、交付前自检脚本。 + 如果用户明确要的是网页翻页式 HTML 演示而不是 .pptx 文件，不要用本技能。 +metadata: + display-name: PPT 制作（BiSheng 适配） +--- + +# 在 BiSheng 里做 PPTX + +## 0. 开工纪律 + +**这一轮只读文档，不要在同一轮里并行调用别的工具。** 读完本文件（必要时再读 references）之后， +下一轮才开始动手。曾经发生过模型把「读 SKILL.md」和「产出交付物」放进同一轮并行调用， +结果技能等于没读、产出完全跑偏。 + +**本技能要求已勾选代码执行器（`bisheng_code_interpreter`）。** 没有它就无法生成 .pptx —— 这种情况下 +直接告诉用户「请在工具里勾选代码执行器后重试」，不要用 export_docx / export_pdf 拿 Word 或 PDF 顶替。 + +## 1. 环境事实（照做，不要试探） + +| 项 | 事实 | +|---|---| +| **生成方式** | **只有 `python-pptx`**。它是后端 `pyproject.toml` 的正式依赖（main / 2.6 / 3.0 各线都有），`import pptx` 直接可用 | +| 不存在的东西 | Node / npm / `pptxgenjs`、`markitdown`、`defusedxml`、`pdfplumber`/`pdfminer`/`PyPDF2`、`pdftoppm`、`zip`/`unzip` | +| 存在但本技能不用 | `pandoc`（发布镜像装了 3.6.4 在 `/usr/bin`）。它能 `-o out.pptx`，但只会产出「标题＋项目符号」的裸版式：配色、版式、图表、图片位置全不可控，做出来必是"一眼 AI"的默认样式。要做能交付的 PPT 一律走 `python-pptx` | +| **前提** | 本技能依赖包内脚本（`skills/bisheng-pptx/scripts/*.py`），只在默认的**本地执行器**下成立。若部署切到 **E2B 沙箱**：`skills/` 不进沙箱、单次上限降到 300 秒 —— 此时脚本调用会 `FileNotFoundError`。**改走纯 python-pptx 内联写法**（cookbook 的片段全部可用，只是不能 `import pptx_helpers`，把需要的函数抄进构建脚本），跳过自检脚本改为自己肉眼核对，并告知用户"当前环境无法运行技能自带的自检脚本" | +| 其它可用库 | Pillow（图片）、PyMuPDF(`fitz`)（读 PDF/渲染）、matplotlib（图表图片）、pandas/numpy、openpyxl、python-docx、lxml | +| 禁止 | `pip install`、`npm install`、任何联网假设（生产多为离线内网） | +| 工作目录 | 执行器 cwd = 工作区根，**一律用相对路径** | +| `output/` | 唯一交付区，已自动创建 | +| `scratch/` | 中间产物区，**不会交付**，需自己 `os.makedirs` | +| `uploads/` | 用户上传的原件（模板、素材、资料）在这里 | +| `skills/bisheng-pptx/` | 本技能包，脚本和参考资料在这里，只读 | +| 绝对禁止 | 写 `/output/xxx.pptx` 这种带前导斜杠的路径 —— 文件会被静默丢弃，用户拿不到 | +| 单次执行上限 | 600 秒。构建 + 自检分多次调用，不要挤在一次里 | +| 日志规则 | **成功时只回传 stdout，stderr 被丢弃** → 一切诊断信息用 `print()`，不要只靠 warning | +| 可见性 | 执行器写完会把产物同步到工作区，之后 `ls`/`read_file` 一般能看到。但判成功看**执行结果**：`exitcode 0` + 日志确认写成功即视为已产出，**不要反复找文件、更不要重做一遍** | +| 轮次 | 最后两轮代码执行器会被摘除 → PPT 必须尽早产出，不要拖到收尾 | + +### 1.1 先探一次环境（第一次执行代码时顺手做，只花一轮） + +```python +import os, shutil +try: + import pptx + print("python-pptx OK", getattr(pptx, "__version__", "")) +except ImportError: + print("python-pptx MISSING") +print("skills 可见:", os.path.isdir("skills/bisheng-pptx/scripts")) +print("soffice:", shutil.which("soffice") or shutil.which("libreoffice") or "无（只影响预览渲染，不影响生成）") +``` + +- `python-pptx MISSING`：正常部署不会出现（它是后端的正式依赖，已在 116 / 180 等环境实测存在）。 + 真遇到就是这套环境被裁剪过 —— **不要 `pip install`**（共享的离线环境，装了会污染所有租户）。 + 直接告诉用户「当前环境缺少 python-pptx，无法生成 .pptx，需要运维在后端环境补装」， + 并问他是否接受改为其它形式的交付物。不要假装做出来了。 +- `skills 可见: False`：说明跑在 E2B 沙箱里（见 §1 的「前提」行）。本包的三个脚本一律调不动， + **不要反复重试路径** —— 直接改走纯 python-pptx 内联写法，自检改为自己核对，并把这个限制告诉用户。 +- `soffice` 没有、或后面渲染时报「无法加载源文件」：说明这台机器的 LibreOffice 没装 Impress 组件。 + **只影响 §5 的可选预览渲染，不影响 .pptx 的生成与交付** —— 跳过看图那一步，以体检结果为准即可。 + +## 2. 选路线 + +| 情况 | 做法 | +|---|---| +| 用户没给模板，要一份新 PPT | §3 从零创建 | +| 用户上传了 .pptx/.potx 模板，或说「按这个样式/模板做」 | §4 套用模板（**优先级最高，别自己另起炉灶**） | +| 用户上传了已有 PPT 要改内容 | 先 §5 的 `inspect_deck.py` 把内容读出来，再按 §4 的方式打开原文件改写 | +| 用户要的是网页翻页 HTML 演示 | 不属于本技能，按常规交付方式做 | + +## 3. 从零创建 + +**第 1 步 · 定结构**。先把大纲写到 `scratch/outline.md`（**不要写进 `output/`**，否则它会取代 PPT 成为 +用户看到的头条交付物）。10–15 页是常见规模：封面 / 目录 / 若干内容页 / 结尾页。 + +**第 2 步 · 定视觉**。选一套与主题相称的配色和版式节奏，细节读 +`/skills/bisheng-pptx/references/design-zh.md`。中文商务、党政国企、科技产品各有惯用调性，不要一律深蓝。 + +**第 3 步 · 写构建脚本**。用 `write_file` 把完整脚本写到 `scratch/build_deck.py`， +**不要把整段代码塞进代码执行器的参数里** —— 参数过长会被截断，导致反复重试却总是差一截。 +写文件工具产生的文件对执行器是可见的。python-pptx 的具体写法读 +`/skills/bisheng-pptx/references/pptx-cookbook.md`（画布尺寸、文本框、项目符号、表格、原生图表、图片、 +中文字体设置，都有可直接抄的片段）。 + +**第 4 步 · 执行**： + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "scratch/build_deck.py"], capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +> 为什么不直接写 `python scratch/build_deck.py`：`python` 在 PATH 里未必是后端那个解释器， +> 用 `sys.executable` 才能保证跑在装了 python-pptx 的环境里。**下面所有脚本调用都用这个写法。** + +**第 5 步 · 自检并返修**（§5）。返修时用 `edit_file` 定点改 `scratch/build_deck.py` 再重跑， +不要每次重写整份脚本。 + +## 4. 套用用户模板 + +**第 1 步 · 探版式**： + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-pptx/scripts/probe_template.py", "uploads/模板.pptx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +它会打印画布尺寸、主题配色与字体、每个版式的索引与占位符 idx、以及模板自带的页。 + +**第 2 步 · 以模板为基底生成**： + +- `prs = Presentation("uploads/模板.pptx")` —— **打开模板本身**，不要 `Presentation()` 空开再仿色。 + 这样母版、主题色、字体、页眉页脚全部自动继承。 +- `slide = prs.slides.add_slide(prs.slide_layouts[i])`，`i` 用第 1 步打印的索引。 +- 填占位符：用本技能包的 **`fill_text(shape, "文字")`**（`pptx_helpers`，见 cookbook §7）， + 它保留模板给这个占位符设定的字号、字色和项目符号。 + **不要用 `text_frame.text = "..."`** —— 那会把整段塌成一个无格式 run，模板的样式全丢。 +- **模板自带的示例页要删掉**（cookbook 有删除页的片段）。删页放在所有内容写完之前做，避免误删刚写的页。 +- 模板里的占位图形若用不到就整组删除，不要只清空文字 —— 会留下孤零零的空框。 +- 保存到 `output/`，扩展名保持 `.pptx`。 + +**注意**：模板文件是二进制，**不要用 `read_file` 去读它**（会被拦截），只能由代码执行器打开。 + +## 5. 交付前自检（必做） + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-pptx/scripts/inspect_deck.py", "output/xxx.pptx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +输出分两段： + +- **内容**：逐页文本 + 备注。用它核对错字、顺序、缺漏 —— 这是 `markitdown` 在本环境的替代品。 +- **体检**：ERROR 必须修完再交付；WARN 逐条复核；INFO 是设计建议。检查覆盖文字溢出、 + 自动撑高的框会压到谁、关闭自动换行导致的截断（ERROR）、超出画布、文字区域重叠、 + 文字压在装饰线/小图形上（WARN）、字号过小、贴边、残留占位符（XXX/待填/"单击此处"）、 + 空占位符（WARN）、整页无视觉元素。 + +**体检阈值比 §7 排版底线松一档**，只在明显违规时出声（例如字号 ERROR 在 8pt 才触发， +而规范要求正文 14–18pt）。**没报 ERROR ≠ 符合规范** —— 排版仍按 §7 和 design-zh 自己把关。 + +改完重新生成，再跑一次，直到「结论: 通过」。 + +**可选 · 看渲染图**： + +> ⚠️ 仅在你确知当前模型支持读图时才做。渲染出的 PNG 会被编成真正的 base64 图片块发给模型厂商， +> 而 BiSheng 默认的 Qwen/dashscope 通道**已知不接收 base64 图片** —— 读图很可能直接失败， +> 甚至中断本次请求。拿不准就跳过，以体检结果为准。 + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-pptx/scripts/render_deck.py", "output/xxx.pptx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +它把每页渲染成 `scratch/preview/<名字>/slide-N.png`，再用 `read_file` 逐张查看。 +**渲染用的中文字体只有文泉驿正黑**，和用户 PowerPoint 里的实际字体宽度不同 —— +预览里的文字松紧只作参考，容器留约 10% 余量即可，不要为了预览效果反复微调字号。 +如果环境里没有 LibreOffice，脚本会直说，跳过这一步、以体检结果为准即可。 + +## 6. 交付纪律 + +- `output/` 里**只放最终的 `.pptx`**。大纲、构建脚本、预览图、中间版本一律放 `scratch/`。 + （同时放一个 `.md` 会让它顶掉 PPT 成为用户看到的头条文件。） +- 文件名用有意义的中文名，如 `output/思源电气企业介绍.pptx`。 +- 收尾时如实说明做了什么、多少页、用了什么风格。**不要声称生成了实际不存在的文件** —— + .pptx 不在系统的幻影交付物检测清单里，写错了没人兜底。 +- 用户拿到的是可下载的 .pptx 文件（当前前端不支持在线预览 PPT），收尾话术不要说「点击预览」。 + +## 7. 排版底线（细则见 references/design-zh.md） + +- 标题 32–44pt 加粗，小标题 20–24pt，正文 14–18pt，注释 10–12pt；正文不要小于 12pt。 +- 每页留 ≥0.5 英寸边距；内容块之间 0.3–0.5 英寸，全篇统一。 +- 除封面外，每页都该有一个视觉元素（图表 / 图形 / 图标 / 表格），不要通篇「标题 + 三行要点」。 +- 正文左对齐，只有标题居中。 +- 字体写「微软雅黑」「黑体」「等线」这类用户端一定有的中文字体（渲染由用户的 PowerPoint 完成）。 +- **一条装饰性线条都不要**：标题上下的横线、章节编号旁的竖线、页面底部的横贯细线、 + 页眉页脚色带、侧边色条、卡片单边描边，全部不要。这是最常被违反的一条， + 既是"一眼 AI"的签名，也是自检里「文字压在装饰线上」的主要来源。用留白和字号层级做分隔。 +- 文字绝不允许溢出容器；放不下就精简文案或换版式，不要一味缩字号。 + +## 8. 绝不要做的事 + +- ❌ 写 `require('pptxgenjs')` 或任何 Node 脚本 —— 装不上，`npm install` 也会失败。 +- ❌ 跑 `markitdown` / `soffice` 命令行做内容 QA —— 用 §5 的两个脚本。 +- ❌ `pip install` 任何东西。 +- ❌ 用绝对路径 `/output/...`。 +- ❌ 把 PPT 降级成 Word/PDF/Markdown 交付。用户要的是 .pptx。 +- ❌ 因为 `ls` 看不到刚生成的文件就重做一遍。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/references/design-zh.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/references/design-zh.md new file mode 100644 index 0000000000..effbfb1edb --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/references/design-zh.md @@ -0,0 +1,114 @@ +# 中文商务 PPT 排版规范 + +面向 16:9 画布（13.333 × 7.5 英寸）。所有数值都可以直接用作 `Inches()` 参数。 + +## 一、开工前定三件事 + +1. **主色要贴题**。把你的配色换到另一个完全不同主题的 PPT 上如果照样成立，说明选得不够具体。 + 电力设备用深红金或电力蓝，环保能源用墨绿，金融用藏青，科技产品用深空灰＋高饱和点缀。 +2. **色彩要有统治关系**。一个主色占 60–70% 视觉重量，1–2 个辅色，一个高饱和强调色只用于关键数字和重点。 + 三色平均分配 = 廉价感。 +3. **定一个视觉母题并贯穿全篇**。比如「圆角卡片」「色块编号」「左侧竖排章节字」，每页都出现同一个元素。 + ⚠️ 不要把「标题下划线」或「顶部色条」当母题 —— 见第六节。 + +## 二、配色方案（可直接取用） + +| 场景 | 主色 | 辅色 | 强调色 | 浅背景 | 正文色 | +|---|---|---|---|---|---| +| 央国企 / 集团宣传（稳重红金） | `8B1A2B` 深红 | `C9A84C` 金 | `A8232F` 正红 | `F7F4EF` 米白 | `2B2B2B` | +| 党政机关 / 汇报（庄重红） | `9E1F24` 中国红 | `D4B366` 暖金 | `7A1519` 暗红 | `FFFFFF` | `1F1F1F` | +| 电力 / 基建（工程蓝） | `123A6B` 深蓝 | `2E6DA4` 中蓝 | `F2A93B` 琥珀 | `F2F5F9` | `24303F` | +| 能源 / 环保（墨绿） | `1E4D3B` 墨绿 | `6FA287` 苔绿 | `D8A657` 麦金 | `F4F7F4` | `223129` | +| 制造 / 装备（钢灰蓝） | `2C3E50` 钢灰 | `5D7891` 雾蓝 | `E67E22` 橙 | `F5F6F7` | `1C2833` | +| 科技 / 数字化（深空） | `121C2E` 深空 | `2F4A72` 靛 | `35D0BA` 青绿 | `FFFFFF` | `1A2233` | +| 金融 / 投资（藏青） | `1B2A4A` 藏青 | `44618C` 灰蓝 | `C0A062` 香槟金 | `F6F7FA` | `20293A` | +| 医疗 / 健康（青白） | `10656D` 孔雀蓝 | `56A3A6` 浅青 | `E8833A` 橘 | `F3F8F8` | `1D3235` | + +用法：深色背景用于封面、章节页、结尾页；浅背景用于内容页 —— 形成「深—浅—深」的三明治结构。 +或者全篇深色，做出高端感，但要保证文字对比度足够。 + +## 三、版式库 + +给出的坐标是 16:9 画布上的建议值（单位英寸）。 + +**封面**：整页主色铺底。主标题 40–44pt 加粗，位于左侧 `(1.0, 2.6)`，宽 9.0；副标题 18–20pt 在其下 0.35； +底部 `(1.0, 6.4)` 放单位名与日期，14pt，透明度感（用辅色）。封面可以只有文字。 + +**目录**：左侧 40% 放「目录 / CONTENTS」大字，右侧 60% 竖排 4–8 个条目， +每条「两位数编号（强调色，24pt）＋ 标题（20pt）」，行距 0.65 英寸。 + +**章节页**：主色铺底，居中或左对齐一个大编号（60–72pt，辅色）＋ 章节名（32pt，白）。 + +**要点页（最常用）**：标题在 `(0.8, 0.55)`，36pt 加粗；内容区从 y=1.75 开始。 +不要一列到底 —— 用下面任一种： +- **两栏**：左栏文字 `(0.8, 1.75, 5.6, 4.6)`，右栏图片/图表 `(6.9, 1.75, 5.6, 4.6)`。 +- **卡片网格**：2×2 或 1×3 圆角矩形，每张卡 `3.9 × 2.1`，间距 0.35，卡内「小标题 20pt 加粗 ＋ 说明 14pt」。 +- **图标行**：每行「彩色圆形（0.55 直径）＋ 加粗小标题 ＋ 说明」，行距 1.1。 +- **半幅出血图**：图片占右半页 `(6.67, 0, 6.67, 7.5)`，文字在左半页留 1.0 边距。 + +**数据页**：大数字 54–72pt 加粗（强调色），单位与说明 12–14pt 在其下；一行放 3–4 个。 +标签的 y 坐标必须按数字字号算出来（数字框高 ≈ 字号 × 1.35），别凭感觉贴 —— 这是最常见的 +"大字压住小标签"来源，cookbook §2.1 有可直接用的写法。 +或用原生图表（见 cookbook），图表占 `(1.0, 1.9, 11.3, 4.5)`。 + +**时间轴 / 流程**：一条水平线 y=3.9，等距节点圆点，节点上下交替放「年份（加粗）＋ 事件（14pt）」， +一页不超过 6 个节点。 + +**结尾页**：与封面同色同构，一句「感谢聆听」＋ 单位名 ＋ 联系方式（14pt）。 + +## 四、字号与间距 + +| 元素 | 字号 | 说明 | +|---|---|---| +| 封面主标题 | 40–44pt 加粗 | | +| 页面标题 | 32–36pt 加粗 | 与正文至少拉开两倍差距 | +| 小标题 / 卡片标题 | 20–24pt 加粗 | | +| 正文 | 14–18pt | 低于 12pt 视为不可读 | +| 图表标签 / 注释 | 10–12pt | 用中性灰 `6B7280` | +| 大数字 | 54–72pt 加粗 | 一页最多 4 个 | + +- 页面四周留白 ≥0.5 英寸（顶部标题区可到 0.55）。 +- 内容块之间统一 0.3 或 0.5 英寸，**全篇只用一种**，不要随机。 +- 正文行距 1.2–1.4；项目符号之间用「段后距」而不是加大行距。 +- 一页正文不超过 6 行、每行不超过 24 个汉字；超了就拆页或换版式。 + +## 五、字体 + +写进 .pptx 的字体名由**用户电脑上的 PowerPoint/WPS 渲染**，不是本环境渲染。因此： + +- **安全中文字体**：微软雅黑、黑体、宋体、等线（Office 自带或 Windows 自带，WPS 也有）。 + 正文优先「微软雅黑」；需要庄重感时标题用「黑体」或「方正小标宋」（后者非必装，作为次选写在同一处即可）。 +- **不要**用只在渲染机上存在的字体（如文泉驿正黑），也不要用 Aptos。 +- 中文字体必须同时写 ``，否则中文会回退到主题字体 —— 用 `pptx_helpers.set_font()`，不要只设 `font.name`。 +- 英文/数字可以配一款无衬线（Arial / Calibri），与中文形成层次。 +- **本地渲染预览用的是文泉驿正黑**，字宽与微软雅黑不同 —— 预览里文字的松紧只作参考，容器留约 10% 余量。 + +## 六、禁忌（这些是"一眼 AI"的特征） + +### 6.0 装饰性线条：一条都不要（最常被违反的一条） + +包括但不限于：标题**下方**的短横线、标题**上方**的分隔线、章节编号旁的竖线、 +页面底部的横贯细线、页眉页脚色带、页面某一侧的竖色条、卡片的单边描边。 +这些线条不承载任何信息，是"AI 做的 PPT"最明显的签名；而且它们经常和正文撞在一起 +（自检脚本里「文字压在装饰线上」这条 WARN，绝大多数就是这么来的）。 + +要做视觉分隔，用这些替代：留白、背景色块（有面积、承载内容的那种）、 +字号与字重的层级差、把标题放到深色区里。 + +- ❌ 标题下面加一条装饰性短横线 / 色条。 +- ❌ 满幅页眉色带、页面左侧竖条、卡片单边描边这类"装饰性色带"。 +- ❌ 每页都是同一个「标题 + 三行要点」，从头到尾不换版式。 +- ❌ 正文居中对齐（只有标题、封面、大数字可以居中）。 +- ❌ 不分主次的彩虹配色，或默认深蓝一路到底。 +- ❌ 米色/奶油色背景当默认底色（`F5F5DC`、`FAF0E6` 这类）；没有指定就用白或主色。 +- ❌ 浅色文字压浅色底、深色图标压深色底这种低对比组合。 +- ❌ 纯文字页（封面除外）。 +- ❌ 文字溢出容器或被裁切 —— 这是最容易被一眼看到的缺陷。 + +## 七、图表规范 + +- 能用原生图表就用原生图表（可在 PowerPoint 里继续编辑），只有 PowerPoint 没有的图型（桑基、网络图）才用图片。 +- 默认图表是"裸"的，必须补：标题、数据标签、取自你配色的系列颜色、 + 弱化的网格线（浅灰）、单系列时关掉图例。 +- 坐标轴标签用 10–12pt 中性灰；不要给每个系列都上高饱和色，主系列用强调色、其余用灰阶。 +- 数据要标单位（亿元 / % / 台），别让读者猜。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/references/pptx-cookbook.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/references/pptx-cookbook.md new file mode 100644 index 0000000000..c5b66e7927 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/references/pptx-cookbook.md @@ -0,0 +1,303 @@ +# python-pptx 配方（BiSheng 环境） + +可直接抄进 `scratch/build_deck.py`。所有片段只依赖 `python-pptx`， +需要 XML 的几处统一走本技能包的 `pptx_helpers`。 + +## 0. 骨架 + +```python +import sys +sys.path.insert(0, "skills/bisheng-pptx/scripts") # 本技能包的 helper + +from pptx import Presentation +from pptx.util import Inches, Pt, Emu +from pptx.dml.color import RGBColor +from pptx.enum.shapes import MSO_SHAPE +from pptx.enum.text import PP_ALIGN, MSO_ANCHOR, MSO_AUTO_SIZE +from pptx_helpers import set_font, set_font_all, add_bullet, no_bullet, delete_slide, fill_text + +W, H = Inches(13.333), Inches(7.5) # 16:9 +prs = Presentation() +prs.slide_width, prs.slide_height = W, H +BLANK = prs.slide_layouts[6] + +# 配色（从 design-zh.md 选一套） +PRIMARY = RGBColor(0x8B, 0x1A, 0x2B) +SECONDARY = RGBColor(0xC9, 0xA8, 0x4C) +ACCENT = RGBColor(0xA8, 0x23, 0x2F) +BG_LIGHT = RGBColor(0xF7, 0xF4, 0xEF) +INK = RGBColor(0x2B, 0x2B, 0x2B) +FONT = "微软雅黑" + +# ... 逐页构建 ... + +import os +os.makedirs("output", exist_ok=True) +prs.save("output/公司介绍.pptx") +print("saved output/公司介绍.pptx", "slides:", len(prs.slides._sldIdLst)) +``` + +> **⚠️ 16:9 的坑**：python-pptx 内置模板是 4:3（10×7.5in）。改了 `slide_width` 之后， +> 内置版式（`slide_layouts[0]`「Title Slide」等）里的占位符**不会**跟着变宽 —— 用它们排出来的页 +> 内容会挤在左侧 3/4。**从零创建时统一用 `slide_layouts[6]`（Blank）自己摆位**。 +> 只有在套用用户模板时才用模板自己的版式（那些是按模板画布做好的）。 + +## 1. 背景与色块 + +```python +def add_slide(bg=None): + slide = prs.slides.add_slide(BLANK) + if bg is not None: + slide.background.fill.solid() + slide.background.fill.fore_color.rgb = bg + return slide + +cover = add_slide(PRIMARY) # 深色封面 +page = add_slide(BG_LIGHT) # 浅色内容页 +``` + +半幅色块 / 卡片： + +```python +card = slide.shapes.add_shape(MSO_SHAPE.ROUNDED_RECTANGLE, + Inches(0.8), Inches(2.0), Inches(3.9), Inches(2.1)) +card.fill.solid(); card.fill.fore_color.rgb = RGBColor(0xFF, 0xFF, 0xFF) +card.line.fill.background() # 无边框（比描边干净） +card.shadow.inherit = False # 关掉默认阴影，默认阴影很脏 +``` + +## 2. 文本框（推荐统一用这个函数） + +```python +def add_text(slide, text, x, y, w, h, size=16, bold=False, color=INK, + align=PP_ALIGN.LEFT, anchor=MSO_ANCHOR.TOP, line_spacing=1.25): + box = slide.shapes.add_textbox(Inches(x), Inches(y), Inches(w), Inches(h)) + tf = box.text_frame + tf.word_wrap = True + tf.auto_size = MSO_AUTO_SIZE.NONE # 固定框高，溢出才能被自检发现 + tf.vertical_anchor = anchor + tf.margin_left = tf.margin_right = Inches(0) # 与其它元素对齐时必须清零 + tf.margin_top = tf.margin_bottom = Inches(0) + lines = text if isinstance(text, list) else [text] + for i, line in enumerate(lines): + para = tf.paragraphs[0] if i == 0 else tf.add_paragraph() + para.alignment = align + para.line_spacing = line_spacing + run = para.add_run() + run.text = line + set_font(run, FONT, size_pt=size, bold=bold, color=color) + return box +``` + +要点： + +- `auto_size = NONE` 很重要。`add_textbox()` 默认是"自动调整高度"，框会在渲染时被文字撑高、 + 压到下方元素，而且自检只能给出 WARN。固定框高之后溢出是硬错误，能被准确报出来。 +- 文本框自带内边距，和线条/图形对齐时把四个 margin 清零，否则文字会莫名内缩 0.1 英寸。 +- 中文字体必须用 `set_font()`（同时写 latin 和 ea），只设 `run.font.name` 对中文无效。 + +## 2.1 大数字 + 标签（最容易撞在一起的组合） + +60–72pt 的数字很高，标签紧跟在下面时极易被数字框压住 —— 渲染出来就是"标签被大字盖掉一半"。 +**按字号算出数字框的高度，再把标签放到它下面**，不要凭感觉给 y 坐标： + +```python +STATS = [("212.05", "2025 年营收（亿元）"), ("31.63", "归母净利润（亿元）"), + ("54.35%", "净利润同比"), ("37.18%", "营收同比")] + +STAT_PT = 60 +stat_h = STAT_PT * 1.35 / 72 # pt → 英寸，1.35 留行高余量 +gap = 0.12 +col_w = 2.8 +for i, (value, label) in enumerate(STATS): + x = 0.8 + i * (col_w + 0.25) + add_text(slide, value, x, 1.9, col_w, stat_h, size=STAT_PT, bold=True, color=PRIMARY) + add_text(slide, label, x, 1.9 + stat_h + gap, col_w, 0.4, size=13, color=INK) +``` + +- 一行最多 4 组；每组宽度要放得下最长的那个标签，放不下就缩短标签或减少组数。 +- 数字里带 `%` 的会更宽，`col_w` 要按最宽的那个数字算，否则 `%` 会自己换行到第二行、把下面的内容全顶乱。 +- 自检里出现「文字区域重叠」的 ERROR，十有八九就是这一处没按上面的方式算 y。 + +## 3. 项目符号 + +```python +box = add_text(slide, ["开关设备", "变压器类产品", "保护与自动化"], 0.8, 1.9, 5.6, 3.0, size=16) +for para in box.text_frame.paragraphs: + add_bullet(para, "▪") # 不要在文字里手打 •，会出现双重符号 + para.space_after = Pt(10) # 用段后距控制间隔，不要靠加大行距 +``` + +需要去掉继承来的符号（例如整段叙述性文字）时用 `no_bullet(para)`。 + +## 4. 表格 + +```python +rows, cols = 4, 3 +tbl_shape = slide.shapes.add_table(rows, cols, Inches(1.0), Inches(2.0), Inches(11.3), Inches(2.6)) +table = tbl_shape.table +table.columns[0].width = Inches(3.5) +table.columns[1].width = Inches(3.9) +table.columns[2].width = Inches(3.9) + +data = [["指标", "2024", "2025"], ["营业收入", "118.5 亿", "140.0 亿"], + ["净利润", "12.4 亿", "15.8 亿"], ["研发投入", "6.1 亿", "7.5 亿"]] +for r, row in enumerate(data): + table.rows[r].height = Inches(0.55) + for c, value in enumerate(row): + cell = table.cell(r, c) + cell.text = value # 表格用 cell.text 没问题 + cell.vertical_anchor = MSO_ANCHOR.MIDDLE + para = cell.text_frame.paragraphs[0] + para.alignment = PP_ALIGN.CENTER if c else PP_ALIGN.LEFT + header = r == 0 + for run in para.runs: + set_font(run, FONT, size_pt=14 if not header else 15, bold=header, + color=RGBColor(0xFF, 0xFF, 0xFF) if header else INK) + cell.fill.solid() + cell.fill.fore_color.rgb = PRIMARY if header else RGBColor(0xFF, 0xFF, 0xFF) +``` + +## 5. 原生图表（能用原生就别贴图片） + +```python +from pptx.chart.data import CategoryChartData +from pptx.enum.chart import XL_CHART_TYPE, XL_LABEL_POSITION + +data = CategoryChartData() +data.categories = ["2023", "2024", "2025"] +data.add_series("营业收入", (98.2, 118.5, 140.0)) + +frame = slide.shapes.add_chart(XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1.0), Inches(1.9), Inches(11.3), Inches(4.5), data) +chart = frame.chart +chart.has_title = True +chart.chart_title.text_frame.text = "营业收入（亿元）" +chart.has_legend = False # 单系列不需要图例 + +plot = chart.plots[0] +plot.gap_width = 120 +plot.has_data_labels = True +labels = plot.data_labels +labels.number_format = "0.0" +labels.number_format_is_linked = False +labels.position = XL_LABEL_POSITION.OUTSIDE_END +labels.font.size = Pt(12) + +series = chart.series[0] +series.format.fill.solid() +series.format.fill.fore_color.rgb = PRIMARY + +chart.font.size = Pt(12) # 轴标签 +chart.font.color.rgb = RGBColor(0x6B, 0x72, 0x80) +``` + +- **堆积柱状图**的数据标签位置只能用 `CENTER` / `INSIDE_END` / `INSIDE_BASE`， + 用 `OUTSIDE_END` 会生成 PowerPoint 打不开的文件。 +- 图表里的中文若在某些环境显示异常，改用第 6 节的 matplotlib 出图贴入。 + +## 6. 图片 / matplotlib 出图 + +```python +slide.shapes.add_picture("uploads/厂区.jpg", Inches(6.9), Inches(1.75), width=Inches(5.6)) +# 只给 width 或 height 会等比缩放；两个都给会拉伸变形 +``` + +> **⚠️ 等比缩放要自己核算高度**：只给 `width` 时高度按原图比例算出来，很容易顶出画布底部。 +> 先算 `高 = 宽 × 原图高/原图宽`，确认 `y + 高 ≤ 7.5 - 0.4`。放不下就减小 `width`。 +> （自检脚本会把越界报成 ERROR，但提前算一步能省一轮返工。） + +matplotlib 生成图片（放 `scratch/`，不要放 `output/`）： + +```python +import os +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib import font_manager + +# 选一个这台机器上真实存在的中文字体，否则中文全是方块。 +# 执行器会自动注入 family="WenQuanYi Zen Hei"，但有的部署装的是 Noto CJK —— 那条注入就落空了。 +_installed = {f.name for f in font_manager.fontManager.ttflist} +for _cand in ("WenQuanYi Zen Hei", "Noto Sans CJK SC", "Source Han Sans SC", "Microsoft YaHei", "SimHei"): + if _cand in _installed: + matplotlib.rc("font", family=_cand) + break +else: + print("⚠️ 没有可用中文字体，图里改用英文标签") +matplotlib.rcParams["axes.unicode_minus"] = False # 负号也会变方块 + +os.makedirs("scratch", exist_ok=True) +fig, ax = plt.subplots(figsize=(7.2, 3.6), dpi=200) # 2:1 +ax.bar(["2023", "2024", "2025"], [98.2, 118.5, 140.0], color="#8B1A2B") +ax.set_ylabel("亿元") +fig.tight_layout() +fig.savefig("scratch/revenue.png", transparent=True) +plt.close(fig) + +# 2:1 的图放 10 英寸宽 → 高 5.0；1.9 + 5.0 = 6.9 < 7.5，安全 +slide.shapes.add_picture("scratch/revenue.png", Inches(1.67), Inches(1.9), width=Inches(10.0)) +``` + +> 执行器会在最后一行 `import matplotlib` 之后自动插入 `matplotlib.rc("font", family="WenQuanYi Zen Hei")`。 +> 上面那段挑字体的代码写在它后面，所以会覆盖它 —— 这正是我们要的：某些部署装的是 Noto CJK 而不是文泉驿， +> 硬写 WenQuanYi 会落空，日志里会刷 `Glyph xxxxx missing from font(s) DejaVu Sans`，图上中文全是方块。 +> 看到这条警告就说明字体没选对，回到这一段。 + +## 7. 套用模板：按占位符填充 + +```python +prs = Presentation("uploads/模板.pptx") # 打开模板本身，继承母版与主题 +slide = prs.slides.add_slide(prs.slide_layouts[3]) # 索引来自 probe_template.py + +fill_text(slide.placeholders[0], "核心业务板块") # 标题 +fill_text(slide.placeholders[1], ["开关设备", "变压器", "储能"]) # 正文，多行 +``` + +- `fill_text()` 会保留模板给这个占位符设定的字号、字色、项目符号； + **绝不要用 `placeholder.text_frame.text = "..."`**，那会把整段塌成一个无格式 run。 +- 用不到的占位符要删掉（下一节），留着会在成品里显示"单击此处添加文本"。 +- 模板里成组的示例元素（如四个人的头像＋姓名）如果你只有三条内容， + 要把第四组**整组删除**，不是只清空文字。 + +删除形状 / 删除页： + +```python +shape._element.getparent().remove(shape._element) # 删一个形状 +delete_slide(prs, 5) # 删一页（从大索引往小删） +``` + +## 8. 演讲者备注 + +```python +slide.notes_slide.notes_text_frame.text = "这一页强调三个数字：140 亿、25%、60 国。" +``` + +备注写在备注页，不要写成幻灯片上的文本框。 + +## 9. 保存与自检 + +```python +import os +os.makedirs("output", exist_ok=True) +prs.save("output/思源电气企业介绍.pptx") +print("OK:", os.path.getsize("output/思源电气企业介绍.pptx"), "bytes") +``` + +保存后一定要跑（见 SKILL.md §5）： + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-pptx/scripts/inspect_deck.py", + "output/思源电气企业介绍.pptx"], capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +## 10. python-pptx 做不到的事（别浪费轮次） + +- **复制一张已有幻灯片**：没有这个 API。要"再来一页同样的"，就用同一个 layout 再 `add_slide` 一次。 +- **渐变填充**：`fill.gradient()` 支持有限且容易出错；要渐变就用一张渐变图片当背景。 +- **读取 SVG/EMF**：`add_picture` 会抛 `UnidentifiedImageError`。模板里的矢量图标只能靠"复用带该图标的版式"来保留。 +- **自动缩放文字以适应形状**：`auto_size` 的自动缩字号在 PowerPoint 里才生效，生成时不会真的改字号 —— + 放不下就自己改文案或改布局。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/inspect_deck.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/inspect_deck.py new file mode 100644 index 0000000000..ad9bbe77c4 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/inspect_deck.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +"""Read a .pptx back and report what is actually in it. + +Replaces two tools the official Anthropic pptx skill relies on but that do not +exist in the BiSheng code interpreter: ``markitdown`` (content dump) and +``office/validate.py`` (which needs ``defusedxml``). + +Two sections are printed: + +1. 内容 — every slide's text plus speaker notes, so content QA (typos, wrong + order, missing sections) can be done without opening the file. +2. 体检 — geometric and editorial checks that approximate a visual review: + text overflow, off-canvas shapes, overlap, tiny fonts, thin margins, + leftover placeholder text, text-only slides. + +Overflow is an *estimate*: PowerPoint's real line breaking depends on the font +metrics of the machine that opens the deck, which are unknowable here. The +estimate is deliberately tuned to under-report mild cases and only shout when a +box is clearly too small. + +Usage: + python skills/bisheng-pptx/scripts/inspect_deck.py output/deck.pptx + python skills/bisheng-pptx/scripts/inspect_deck.py output/deck.pptx --checks-only + +Always exits 0, including on a fatal error — see the comment in main(). +""" + +from __future__ import annotations + +import argparse +import math +import os +import re +import sys +import traceback +from dataclasses import dataclass + +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.enum.text import MSO_ANCHOR, MSO_AUTO_SIZE + +# --- tunables ------------------------------------------------------------- +# +# House rule for every threshold below: the checker sits ONE NOTCH LOOSER than +# the authoring spec (SKILL.md §7 and references/design-zh.md), and only shouts +# on a clear violation. Rationale: an ERROR here is a hard delivery gate, so a +# threshold set at the spec value would block decks that are merely at the edge +# of the recommended range (e.g. design-zh allows 10-12pt captions — an ERROR at +# "<12pt" would make a compliant deck unshippable, and the model would learn to +# ignore the report). The spec stays the authority on how to lay a slide out; +# this file only catches what is visibly broken. +# +# Every constant names the clause it derives from. Drift between the two used to +# be invisible precisely because that anchor was missing — keep it, and keep the +# Finding message consistent with the number that actually fires. + +DEFAULT_FONT_PT = 18.0 # technical fallback when the size is inherited and unreadable here +DEFAULT_LINE_SPACING = 1.2 # technical fallback; design-zh section 4 recommends 1.2-1.4 + +# ANCHOR: SKILL.md section 7, "text must never overflow its container"; design-zh +# section 6, same clause. The spec is zero tolerance; this estimate carries about +# +-10% of slack (the font metrics of the opening machine are unknowable), so WARN +# starts at +5% over the box and ERROR at +30%. +OVERFLOW_WARN_RATIO = 1.05 +OVERFLOW_ERROR_RATIO = 1.30 + +# ANCHOR: design-zh section 4, the font-size table — body 14-18pt, chart labels and +# captions 10-12pt, "below 12pt counts as unreadable". WARN fires below the smallest +# size the spec allows for ANY role (10pt), so a legitimate 10-12pt caption stays +# silent; ERROR fires only where no role is readable at all (<8pt). +MIN_FONT_WARN_PT = 10.0 +MIN_FONT_ERROR_PT = 8.0 + +# ANCHOR: design-zh section 4, "keep >= 0.5 inch of margin on all four sides" = 36pt. +# Fires at 0.3in, i.e. only when the shape visibly hugs the edge rather than merely +# sitting below the recommendation. +MARGIN_WARN_PT = 21.6 # 0.3 inch + +# ANCHOR: design-zh section 6 lists overlapping/clipped text among the "obviously AI" +# defects, but gives no number. Text over text is a defect at a much lower threshold +# than text over a shape: a big stat number sitting 30% into its own caption already +# renders unreadable. +OVERLAP_WARN_RATIO = 0.15 +OVERLAP_ERROR_RATIO = 0.30 + +# ANCHOR: design-zh section 6.0, "not a single decorative rule", repeated in SKILL.md +# section 7. A "decoration" is a thin rule (timeline axis, divider) or a small mark +# (node dot, icon). Text running across one of those is a real, visible defect — +# unlike text sitting on a card background, which is the intended design. +DECOR_THIN_PT = 20.0 +DECOR_SMALL_AREA_PT = 2600.0 # ≈ 0.5 in² +DECOR_HIT_RATIO = 0.10 + +PLACEHOLDER_PATTERNS = [ + (re.compile(r"\bx{3,}\b", re.I), "XXX 占位"), + (re.compile(r"lorem|ipsum", re.I), "lorem ipsum"), + (re.compile(r"\bTODO\b|\bTBD\b", re.I), "TODO/TBD"), + (re.compile(r"待填|待补充|此处填写|请输入|占位符"), "中文占位提示"), + (re.compile(r"\{\{.*?\}\}|\[insert|【\s*】"), "模板变量未替换"), + (re.compile(r"单击此处|点击此处添加|Click to edit", re.I), "PowerPoint 默认提示语"), +] + +EMU_PER_PT = 12700 + + +@dataclass +class Finding: + level: str # ERROR / WARN / INFO + slide: int + where: str + message: str + + def render(self) -> str: + loc = f"第{self.slide}页" if self.slide else "整体" + where = f" 「{self.where}」" if self.where else "" + return f"{self.level:<5} {loc}{where} {self.message}" + + +# --- geometry helpers ----------------------------------------------------- + + +def _pt(value) -> float: + """EMU (int or pptx Length) -> points.""" + if value is None: + return 0.0 + return float(value) / EMU_PER_PT + + +def _char_em(ch: str) -> float: + """Rough advance width of one character, in em units of the font size.""" + o = ord(ch) + if ( + 0x4E00 <= o <= 0x9FFF # CJK unified + or 0x3400 <= o <= 0x4DBF # extension A + or 0xF900 <= o <= 0xFAFF # compatibility + or 0x3000 <= o <= 0x303F # CJK punctuation + or 0xFF01 <= o <= 0xFF60 # fullwidth forms + or 0xFFE0 <= o <= 0xFFE6 + or 0x3040 <= o <= 0x30FF # kana + ): + return 1.0 + if ch == " ": + return 0.28 + if ch in "iljItfr().,;:'\"|!": + return 0.32 + if ch.isupper() or ch.isdigit(): + return 0.60 + return 0.53 + + +def _text_em_width(text: str) -> float: + return sum(_char_em(c) for c in text) + + +def _resolve_size_pt(run, para, shape, layout_sizes: dict) -> float: + """Effective font size for a run, walking the inheritance chain we can see.""" + if run is not None and run.font.size is not None: + return run.font.size.pt + if para.font.size is not None: + return para.font.size.pt + if shape.is_placeholder: + idx = shape.placeholder_format.idx + if idx in layout_sizes: + return layout_sizes[idx] + return DEFAULT_FONT_PT + + +def _layout_placeholder_sizes(slide) -> dict: + """idx -> font size (pt) declared on the layout, when it declares one.""" + sizes = {} + try: + placeholders = slide.slide_layout.placeholders + except (AttributeError, KeyError): + return sizes + for ph in placeholders: + if not ph.has_text_frame: + continue + for para in ph.text_frame.paragraphs: + size = para.font.size + if size is None: + for run in para.runs: + if run.font.size is not None: + size = run.font.size + break + if size is not None: + sizes[ph.placeholder_format.idx] = size.pt + break + return sizes + + +def _para_line_height_pt(para, size_pt: float) -> float: + spacing = para.line_spacing + if spacing is None: + return size_pt * DEFAULT_LINE_SPACING + if isinstance(spacing, (int, float)): + return size_pt * float(spacing) + return _pt(spacing) # Length: exact leading + + +def _estimate_text_height_pt(shape, layout_sizes: dict) -> tuple[float, float]: + """(estimated content height, widest single line width) in points.""" + tf = shape.text_frame + width_pt = _pt(shape.width) + avail_w = width_pt - _pt(tf.margin_left) - _pt(tf.margin_right) + wrap = tf.word_wrap is not False + + total = _pt(tf.margin_top) + _pt(tf.margin_bottom) + widest = 0.0 + for para in tf.paragraphs: + runs = list(para.runs) + text = "".join(r.text for r in runs) + sizes = [_resolve_size_pt(r, para, shape, layout_sizes) for r in runs] + size_pt = max(sizes) if sizes else _resolve_size_pt(None, para, shape, layout_sizes) + + em_width = _text_em_width(text) + line_w = em_width * size_pt + widest = max(widest, line_w) + + if not text: + lines = 1 + elif not wrap or avail_w <= 0: + lines = 1 + else: + lines = max(1, math.ceil(line_w / avail_w)) + + total += lines * _para_line_height_pt(para, size_pt) + total += _pt(para.space_before) + _pt(para.space_after) + return total, widest + + +def _effective_text_box(shape, content_h: float, auto_grow: bool) -> tuple[float, float, float, float]: + """The band the text actually occupies, for overlap testing. + + A title placeholder is often far taller than its one line of text; using the + raw frame would make every deck look like it has overlapping shapes. An + auto-grow box is the opposite case — it occupies more than its stored height. + """ + left, top = _pt(shape.left), _pt(shape.top) + width, height = _pt(shape.width), _pt(shape.height) + if auto_grow and content_h > height: + return left, top, width, content_h + if content_h >= height: + return left, top, width, height + anchor = shape.text_frame.vertical_anchor + if anchor == MSO_ANCHOR.MIDDLE: + top += (height - content_h) / 2 + elif anchor == MSO_ANCHOR.BOTTOM: + top += height - content_h + return left, top, width, content_h + + +def _rect_overlap(a, b) -> float: + ax, ay, aw, ah = a + bx, by, bw, bh = b + dx = min(ax + aw, bx + bw) - max(ax, bx) + dy = min(ay + ah, by + bh) - max(ay, by) + if dx <= 0 or dy <= 0: + return 0.0 + return dx * dy + + +# --- traversal ------------------------------------------------------------ + + +def _iter_shapes(shapes, prefix: str = ""): + """Flatten groups; yields (shape, display_name).""" + for shape in shapes: + name = f"{prefix}{shape.name}" + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from _iter_shapes(shape.shapes, prefix=f"{name}/") + else: + yield shape, name + + +def _shape_text(shape) -> str: + if shape.has_text_frame: + return shape.text_frame.text + if getattr(shape, "has_table", False): + rows = [] + for row in shape.table.rows: + rows.append(" | ".join(cell.text.replace("\n", " ") for cell in row.cells)) + return "\n".join(rows) + return "" + + +_MEDIA_TYPES = (MSO_SHAPE_TYPE.PICTURE, MSO_SHAPE_TYPE.CHART, MSO_SHAPE_TYPE.TABLE) + + +def _is_visual(shape) -> bool: + if shape.shape_type in (MSO_SHAPE_TYPE.PICTURE, MSO_SHAPE_TYPE.CHART, MSO_SHAPE_TYPE.TABLE): + return True + if getattr(shape, "has_chart", False) or getattr(shape, "has_table", False): + return True + if shape.shape_type == MSO_SHAPE_TYPE.AUTO_SHAPE: + return True + return False + + +# --- checks --------------------------------------------------------------- + + +def check_slide(slide, index: int, slide_w: float, slide_h: float) -> list[Finding]: + findings: list[Finding] = [] + layout_sizes = _layout_placeholder_sizes(slide) + text_boxes = [] # (rect, name) for overlap + decorations = [] # (rect, name) thin rules / small marks text must not cross + has_visual = False + + for shape, name in _iter_shapes(slide.shapes): + left, top = _pt(shape.left), _pt(shape.top) + width, height = _pt(shape.width), _pt(shape.height) + full_bleed = left <= 1 and top <= 1 and left + width >= slide_w - 1 and top + height >= slide_h - 1 + + if _is_visual(shape): + has_visual = True + + has_text = shape.has_text_frame and shape.text_frame.text.strip() + content_h = 0.0 + widest = 0.0 + auto_grow = False + if has_text: + content_h, widest = _estimate_text_height_pt(shape, layout_sizes) + # add_textbox() writes by default, so most generated + # boxes are auto-grow: the text is never clipped, but the box swells + # downward at render time and can push past the canvas or onto the + # element below. Judge those by the grown height, not the stored one. + auto_grow = shape.text_frame.auto_size == MSO_AUTO_SIZE.SHAPE_TO_FIT_TEXT + effective_h = max(height, content_h) if auto_grow else height + + # off-canvas (using the height the shape will actually occupy) + if not full_bleed: + over_r = (left + width) - slide_w + over_b = (top + effective_h) - slide_h + worst = max(-left, -top, over_r, over_b) + if worst > 1: + grown = " 且文字会把框撑高" if auto_grow and effective_h > height + 1 else "" + findings.append( + Finding( + "ERROR", + index, + name, + f"超出画布 {worst:.0f}pt{grown}（画布 {slide_w:.0f}×{slide_h:.0f}pt）→ 移回画布内", + ) + ) + + if not has_text and not full_bleed and shape.shape_type not in _MEDIA_TYPES: + thin = min(width, height) < DECOR_THIN_PT + small = width * height < DECOR_SMALL_AREA_PT + if (thin or small) and width > 0 and height > 0: + decorations.append(((left, top, width, height), name)) + + if not shape.has_text_frame: + continue + + if not has_text: + if shape.is_placeholder: + findings.append(Finding("WARN", index, name, "占位符是空的 → 填内容或删掉这个占位符")) + continue + + text = shape.text_frame.text + + if height > 0 and not auto_grow: + ratio = content_h / height + if ratio >= OVERFLOW_ERROR_RATIO: + findings.append( + Finding( + "ERROR", + index, + name, + f"文字预计需要 {content_h:.0f}pt，框高只有 {height:.0f}pt" + f"（超出 {(ratio - 1) * 100:.0f}%）→ 精简文字 / 降字号 / 加高框", + ) + ) + elif ratio >= OVERFLOW_WARN_RATIO: + findings.append( + Finding( + "WARN", + index, + name, + f"文字可能放不下（估算 {content_h:.0f}pt vs 框高 {height:.0f}pt）→ 复核这一处", + ) + ) + elif height > 0 and auto_grow and content_h > height * OVERFLOW_WARN_RATIO: + findings.append( + Finding( + "WARN", + index, + name, + f"框设了自动调整高度，文字会把它从 {height:.0f}pt 撑到约 {content_h:.0f}pt" + f"（向下多占 {content_h - height:.0f}pt）→ 确认没有压到下方元素", + ) + ) + + if shape.text_frame.word_wrap is False and widest > width: + findings.append( + Finding( + "ERROR", + index, + name, + f"关闭了自动换行且单行宽 {widest:.0f}pt > 框宽 {width:.0f}pt → 文字会被截断", + ) + ) + + # font size + smallest = None + for para in shape.text_frame.paragraphs: + for run in para.runs: + if not run.text.strip(): + continue + size = _resolve_size_pt(run, para, shape, layout_sizes) + smallest = size if smallest is None else min(smallest, size) + if smallest is not None: + if smallest < MIN_FONT_ERROR_PT: + findings.append( + Finding( + "ERROR", + index, + name, + f"字号 {smallest:.1f}pt 低于可读下限 {MIN_FONT_ERROR_PT:.0f}pt，投影上完全看不清" + f" → 正文改到 14-18pt，注释类不低于 10pt", + ) + ) + elif smallest < MIN_FONT_WARN_PT: + findings.append( + Finding( + "WARN", + index, + name, + f"字号 {smallest:.1f}pt 低于注释类下限 {MIN_FONT_WARN_PT:.0f}pt" + f" → 正文改到 14-18pt，注释类 10-12pt", + ) + ) + + # margins from the canvas edge + if not full_bleed: + gaps = [left, top, slide_w - (left + width), slide_h - (top + height)] + tight = min(gaps) + if 0 <= tight < MARGIN_WARN_PT: + findings.append( + Finding( + "WARN", + index, + name, + f"离画布边缘只有 {tight:.0f}pt，已低于贴边报警线 {MARGIN_WARN_PT:.0f}pt(0.3in)" + f" → 往里挪，规范是四周留 ≥36pt(0.5in)", + ) + ) + + # leftover placeholder copy + for pattern, label in PLACEHOLDER_PATTERNS: + if pattern.search(text): + findings.append(Finding("ERROR", index, name, f"残留{label}：{text.strip()[:40]!r} → 替换成真实内容")) + break + + if not full_bleed: + text_boxes.append((_effective_text_box(shape, content_h, auto_grow), name)) + + # overlap between text bands + for i in range(len(text_boxes)): + for j in range(i + 1, len(text_boxes)): + rect_a, name_a = text_boxes[i] + rect_b, name_b = text_boxes[j] + inter = _rect_overlap(rect_a, rect_b) + if inter <= 0: + continue + smaller = min(rect_a[2] * rect_a[3], rect_b[2] * rect_b[3]) + if smaller <= 0: + continue + ratio = inter / smaller + if ratio >= OVERLAP_ERROR_RATIO: + findings.append( + Finding("ERROR", index, f"{name_a} × {name_b}", f"文字区域重叠 {ratio * 100:.0f}% → 会互相压字") + ) + elif ratio >= OVERLAP_WARN_RATIO: + findings.append( + Finding("WARN", index, f"{name_a} × {name_b}", f"文字区域重叠 {ratio * 100:.0f}% → 复核间距") + ) + + # text running across a rule or a node dot + for text_rect, text_name in text_boxes: + for decor_rect, decor_name in decorations: + inter = _rect_overlap(text_rect, decor_rect) + decor_area = decor_rect[2] * decor_rect[3] + if decor_area <= 0 or inter / decor_area < DECOR_HIT_RATIO: + continue + # Text sitting INSIDE the mark (a number in a circle, a label on a + # badge) is the intended design, not a collision. Only text that + # merely crosses the shape is a defect. + text_area = text_rect[2] * text_rect[3] + if text_area > 0 and inter / text_area > 0.7: + continue + findings.append( + Finding( + "WARN", + index, + f"{text_name} × {decor_name}", + "文字压在装饰线/小图形上 → 把文字移开，或让线/点避开这段文字", + ) + ) + + # The cover is legitimately text-only; every other slide should carry something visual. + if not has_visual and index > 1: + findings.append(Finding("INFO", index, "", "整页只有文字，没有图片/图表/表格/图形 → 考虑加一个视觉元素")) + + return findings + + +# --- output --------------------------------------------------------------- + + +def dump_content(prs) -> None: + print("=== 内容 ===") + for i, slide in enumerate(prs.slides, start=1): + try: + layout_name = slide.slide_layout.name + except (AttributeError, KeyError): + layout_name = "?" + print(f"\n[第{i}页] 版式: {layout_name}") + for shape, _name in _iter_shapes(slide.shapes): + text = _shape_text(shape) + if not text.strip(): + continue + for line in text.splitlines(): + if line.strip(): + print(f" · {line.strip()}") + if slide.has_notes_slide: + notes = slide.notes_slide.notes_text_frame.text.strip() + if notes: + print(f" [备注] {notes}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Dump and health-check a .pptx") + parser.add_argument("pptx", help="路径，相对工作区根，例如 output/deck.pptx") + parser.add_argument("--checks-only", action="store_true", help="只输出体检，不 dump 内容") + args = parser.parse_args() + + if not os.path.exists(args.pptx): + print(f"[FATAL] 文件不存在: {args.pptx}") + print("请用相对工作区根的路径，例如 output/deck.pptx（不要带前导斜杠，也不要用宿主机绝对路径）。") + return 0 + + # Always exit 0, including on a crash: on a non-zero exit the code + # interpreter returns stderr and throws stdout away, which would delete this + # whole report. A traceback printed to stdout is far more useful than that. + try: + run_report(args.pptx, checks_only=args.checks_only) + except Exception: + print("[FATAL] 读取 / 体检 .pptx 时出错：") + traceback.print_exc(file=sys.stdout) + print("常见原因：文件损坏或写了一半、是加密文档、其实是老式 .ppt（python-pptx 只认 .pptx）。") + print("→ 重新跑一次构建脚本生成完整的 .pptx，再跑本脚本；老式 .ppt 请让用户另存为 .pptx。") + return 0 + + +def run_report(path: str, checks_only: bool) -> None: + prs = Presentation(path) + slide_w, slide_h = _pt(prs.slide_width), _pt(prs.slide_height) + + if not checks_only: + dump_content(prs) + + findings: list[Finding] = [] + slide_count = 0 + for i, slide in enumerate(prs.slides, start=1): + slide_count = i + findings.extend(check_slide(slide, i, slide_w, slide_h)) + + print(f"\n=== 体检 === ({slide_count} 页, 画布 {slide_w:.0f}×{slide_h:.0f}pt)") + order = {"ERROR": 0, "WARN": 1, "INFO": 2} + for finding in sorted(findings, key=lambda f: (order[f.level], f.slide)): + print(finding.render()) + + errors = sum(1 for f in findings if f.level == "ERROR") + warns = sum(1 for f in findings if f.level == "WARN") + infos = sum(1 for f in findings if f.level == "INFO") + print(f"\n合计: {errors} ERROR / {warns} WARN / {infos} INFO") + if errors: + print(f"结论: 不通过 —— 有 {errors} 项必须修复。改构建脚本重新生成，再跑一次本脚本。") + else: + print("结论: 通过 —— 无必须修复项。WARN 逐条复核后即可交付。") + print("提示：体检阈值比 SKILL.md §7 / design-zh 的规范松一档，只在明显违规时出声；") + print(" 没报 ERROR 不等于完全合规，排版仍以规范为准。溢出为估算值，不必为 WARN 反复调参。") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/pptx_helpers.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/pptx_helpers.py new file mode 100644 index 0000000000..d7f83edb19 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/pptx_helpers.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +"""The handful of python-pptx operations that need raw OOXML, done correctly. + +Import from a build script running in the code interpreter: + + import sys + sys.path.insert(0, "skills/bisheng-pptx/scripts") + from pptx_helpers import set_font, add_bullet, delete_slide, fill_text + +Everything here is standard python-pptx plus lxml — no extra dependencies. +""" + +from __future__ import annotations + +import copy + +from pptx.oxml.ns import qn +from pptx.util import Inches, Pt + +__all__ = [ + "add_bullet", + "delete_slide", + "fill_text", + "iter_runs", + "no_bullet", + "set_font", + "set_font_all", +] + + +def set_font(run, name: str, size_pt: float | None = None, bold: bool | None = None, color=None) -> None: + """Set a run's font so that **Chinese text actually uses it**. + + ``run.font.name`` only writes ````, which governs Latin characters. + CJK glyphs follow ````; without it PowerPoint falls back to the theme's + East-Asian font and your font choice silently does nothing to the Chinese. + ```` must sit immediately after ```` — appending it anywhere + else produces a file PowerPoint refuses to open. + """ + if size_pt is not None: + run.font.size = Pt(size_pt) + if bold is not None: + run.font.bold = bold + if color is not None: + run.font.color.rgb = color + + rPr = run._r.get_or_add_rPr() + latin = rPr.get_or_add_latin() + latin.set("typeface", name) + ea = rPr.find(qn("a:ea")) + if ea is None: + ea = latin.makeelement(qn("a:ea"), {}) + latin.addnext(ea) + ea.set("typeface", name) + + +def iter_runs(shape): + """Every run in a shape's text frame.""" + if not shape.has_text_frame: + return + for para in shape.text_frame.paragraphs: + yield from para.runs + + +def set_font_all(shape, name: str, size_pt: float | None = None) -> None: + """Apply set_font to every run of a shape (handy after filling a template).""" + for run in iter_runs(shape): + set_font(run, name, size_pt=size_pt) + + +def add_bullet(paragraph, char: str = "•", font: str = "Arial", indent_in: float = 0.25) -> None: + """Give a paragraph a real bullet, with a hanging indent. + + Never type "•" into the text itself — layouts that already supply a bullet + would then render two. ```` must be inserted before ```` + to keep the element order valid. + + ``indent_in`` sets the gap between the bullet and the text (marL with a + negative first-line indent). Without it the glyph sits flush against the + first character and wrapped lines do not align under the text. + """ + pPr = paragraph._p.get_or_add_pPr() + if indent_in: + offset = int(Inches(indent_in)) + pPr.set("marL", str(offset)) + pPr.set("indent", str(-offset)) + for tag in ("a:buNone", "a:buChar", "a:buAutoNum"): + existing = pPr.find(qn(tag)) + if existing is not None: + pPr.remove(existing) + + bu_font = pPr.makeelement(qn("a:buFont"), {"typeface": font}) + bu_char = pPr.makeelement(qn("a:buChar"), {"char": char}) + anchor = pPr.find(qn("a:defRPr")) + if anchor is not None: + anchor.addprevious(bu_font) + anchor.addprevious(bu_char) + else: + pPr.append(bu_font) + pPr.append(bu_char) + + +def no_bullet(paragraph) -> None: + """Suppress an inherited bullet (e.g. a body placeholder used for prose).""" + pPr = paragraph._p.get_or_add_pPr() + for tag in ("a:buChar", "a:buAutoNum", "a:buFont"): + existing = pPr.find(qn(tag)) + if existing is not None: + pPr.remove(existing) + if pPr.find(qn("a:buNone")) is None: + bu_none = pPr.makeelement(qn("a:buNone"), {}) + anchor = pPr.find(qn("a:defRPr")) + if anchor is not None: + anchor.addprevious(bu_none) + else: + pPr.append(bu_none) + + +def delete_slide(prs, index: int) -> None: + """Remove a slide by position, dropping its relationship too. + + python-pptx has no public delete; leaving the rel behind yields a file that + PowerPoint reports as damaged. Delete from the highest index first when + removing several, or the positions shift under you. + """ + slide_ids = prs.slides._sldIdLst + entries = list(slide_ids) + entry = entries[index] + rel_id = entry.get(qn("r:id")) + prs.part.drop_rel(rel_id) + slide_ids.remove(entry) + + +def fill_text(shape, lines) -> None: + """Replace a shape's text while keeping the template's formatting. + + ``text_frame.text = "..."`` collapses the whole frame into one unstyled run, + throwing away the size, colour and bullet the template author set. This + instead rewrites the first run in place and clones that paragraph for any + further lines. + + ``lines`` may be a string (split on newlines) or a list of strings. + """ + if isinstance(lines, str): + lines = lines.split("\n") + lines = list(lines) or [""] + + tf = shape.text_frame + first_p = tf.paragraphs[0]._p + body = first_p.getparent() + + for para in list(body.findall(qn("a:p")))[1:]: + body.remove(para) + + def _write(p_el, text: str) -> None: + runs = p_el.findall(qn("a:r")) + if not runs: + run = p_el.makeelement(qn("a:r"), {}) + text_el = p_el.makeelement(qn("a:t"), {}) + run.append(text_el) + p_el.append(run) + runs = [run] + for extra in runs[1:]: + p_el.remove(extra) + text_el = runs[0].find(qn("a:t")) + if text_el is None: + text_el = runs[0].makeelement(qn("a:t"), {}) + runs[0].append(text_el) + text_el.text = text + if text != text.strip(): + text_el.set(qn("xml:space"), "preserve") + + _write(first_p, lines[0]) + for line in lines[1:]: + clone = copy.deepcopy(first_p) + _write(clone, line) + body.append(clone) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/probe_template.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/probe_template.py new file mode 100644 index 0000000000..e8583fbdc7 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/probe_template.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +"""Describe a .pptx/.potx template so its layouts can be reused instead of guessed at. + +python-pptx cannot duplicate an existing slide, but it *can* add a slide from any +layout the template defines — inheriting that template's master, theme colours, +fonts and placeholder geometry. To do that you need to know which layout index +does what, and which placeholder idx to fill. That is what this prints. + +Usage: + python skills/bisheng-pptx/scripts/probe_template.py uploads/模板.pptx + python skills/bisheng-pptx/scripts/probe_template.py uploads/模板.pptx --layout 3 +""" + +from __future__ import annotations + +import argparse +import os +import sys +import traceback + +from lxml import etree +from pptx import Presentation +from pptx.opc.constants import RELATIONSHIP_TYPE as RT + +EMU_PER_INCH = 914400 +A_NS = "{http://schemas.openxmlformats.org/drawingml/2006/main}" + +COLOR_LABELS = { + "dk1": "深色1/正文色", + "lt1": "浅色1/背景色", + "dk2": "深色2", + "lt2": "浅色2", + "accent1": "强调色1", + "accent2": "强调色2", + "accent3": "强调色3", + "accent4": "强调色4", + "accent5": "强调色5", + "accent6": "强调色6", + "hlink": "超链接", + "folHlink": "已访问链接", +} + + +def _inch(value) -> float: + return 0.0 if value is None else float(value) / EMU_PER_INCH + + +def _ph_label(shape) -> str: + fmt = shape.placeholder_format + try: + type_name = str(fmt.type).split(" ")[0] + except (ValueError, AttributeError): + type_name = "?" + return f"idx={fmt.idx} type={type_name}" + + +def _theme_root(prs): + """The first master's theme part as an lxml element, or None.""" + try: + part = prs.slide_masters[0].part.part_related_by(RT.THEME) + except (KeyError, IndexError): + return None + return etree.fromstring(part.blob) + + +def _color_of(element) -> str: + if element is None: + return "-" + srgb = element.find(f"{A_NS}srgbClr") + if srgb is not None: + return srgb.get("val", "-").upper() + sys_clr = element.find(f"{A_NS}sysClr") + if sys_clr is not None: + return (sys_clr.get("lastClr") or sys_clr.get("val") or "-").upper() + return "-" + + +def dump_theme(prs) -> None: + root = _theme_root(prs) + print("\n=== 主题（新增页会自动继承，不要硬编码覆盖）===") + if root is None: + print(" 读不到 theme part，按默认 Office 主题处理") + return + + scheme = root.find(f".//{A_NS}clrScheme") + if scheme is not None: + print(" 配色:") + for child in scheme: + key = etree.QName(child).localname + print(f" {COLOR_LABELS.get(key, key):<12} {key:<10} #{_color_of(child)}") + + fonts = root.find(f".//{A_NS}fontScheme") + if fonts is not None: + print(" 字体:") + for tag, label in (("majorFont", "标题"), ("minorFont", "正文")): + node = fonts.find(f"{A_NS}{tag}") + if node is None: + continue + latin = node.find(f"{A_NS}latin") + ea = node.find(f"{A_NS}ea") + latin_name = latin.get("typeface", "-") if latin is not None else "-" + ea_name = ea.get("typeface", "") if ea is not None else "" + suffix = f" / 东亚: {ea_name}" if ea_name else " / 东亚: 未指定（会回退到正文字体）" + print(f" {label}: {latin_name}{suffix}") + + +def dump_layouts(prs, only: int | None) -> None: + master = prs.slide_masters[0] + print(f"\n=== 版式清单（共 {len(master.slide_layouts)} 个）===") + print("用法: slide = prs.slides.add_slide(prs.slide_layouts[索引])，再按 idx 填占位符") + for i, layout in enumerate(master.slide_layouts): + if only is not None and i != only: + continue + placeholders = list(layout.placeholders) + print(f"\n [{i}] {layout.name} 占位符 {len(placeholders)} 个") + for ph in placeholders: + geo = f"位置 {_inch(ph.left):.2f},{_inch(ph.top):.2f} 尺寸 {_inch(ph.width):.2f}×{_inch(ph.height):.2f} in" + sample = ph.text_frame.text.strip().replace("\n", " ")[:24] if ph.has_text_frame else "" + sample = f' 示例文字="{sample}"' if sample else "" + print(f" {_ph_label(ph):<28} {geo}{sample}") + + +def dump_slides(prs) -> None: + slides = list(prs.slides) + print(f"\n=== 模板自带的页（共 {len(slides)} 页）===") + if not slides: + print(" 无（纯版式模板，直接 add_slide 即可）") + return + print(" 这些页会原样留在你的成品里 —— 用不到的必须删掉（见 SKILL.md「删除模板自带页」）") + for i, slide in enumerate(slides, start=1): + try: + layout_name = slide.slide_layout.name + except (AttributeError, KeyError): + layout_name = "?" + title = "" + for shape in slide.shapes: + if shape.has_text_frame and shape.text_frame.text.strip(): + title = shape.text_frame.text.strip().splitlines()[0][:30] + break + print(f" 第{i}页 版式={layout_name:<24} 首行文字={title!r}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Probe a pptx/potx template") + parser.add_argument("pptx", help="模板路径，相对工作区根，例如 uploads/模板.pptx") + parser.add_argument("--layout", type=int, default=None, help="只看某一个版式的占位符") + args = parser.parse_args() + + if not os.path.exists(args.pptx): + print(f"[FATAL] 文件不存在: {args.pptx}（用相对工作区根的路径，上传件通常在 uploads/ 下）") + return 0 + + # Always exit 0: a non-zero exit makes the code interpreter return stderr and + # drop stdout, which would hide both this report and the reason it stopped. + try: + describe(args.pptx, args.layout) + except Exception: + print("[FATAL] 解析模板时出错：") + traceback.print_exc(file=sys.stdout) + print("常见原因：文件损坏、是加密文档、或其实是老式 .ppt/.pot（python-pptx 只认 .pptx/.potx）。") + print("→ 请用户另存为 .pptx 后重传；拿不到就放弃套模板，改按 SKILL.md §3 从零创建并告知用户。") + return 0 + + +def describe(path: str, only_layout: int | None) -> None: + prs = Presentation(path) + w_in, h_in = _inch(prs.slide_width), _inch(prs.slide_height) + ratio = "16:9" if abs(w_in / h_in - 16 / 9) < 0.02 else ("4:3" if abs(w_in / h_in - 4 / 3) < 0.02 else "自定义") + print(f"=== 画布 ===\n {w_in:.2f} × {h_in:.2f} in ({ratio}) —— 不要改画布尺寸，沿用模板的") + + dump_theme(prs) + dump_layouts(prs, only_layout) + dump_slides(prs) + + print("\n=== 下一步 ===") + print(" 1) 用 Presentation('该模板路径') 打开它当基底，不要 Presentation() 空开") + print(" 2) 按上面的索引 add_slide，按 idx 定位占位符，赋值 run.text 而不是 text_frame.text") + print(" 3) 删掉模板自带页里用不到的部分，最后另存到 output/") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/render_deck.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/render_deck.py new file mode 100644 index 0000000000..dbf52584d2 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-pptx/scripts/render_deck.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +"""Render a .pptx to one PNG per slide, for visual QA. + +Replaces the official skill's ``soffice.py`` + ``pdftoppm`` chain, neither of +which works here: ``pdftoppm`` is not installed, and a bare ``soffice`` call +fails with "source file could not be loaded" unless it gets its own throwaway +user profile and an absolute input path. The same invocation the repo already +uses in ``knowledge/rag/pipeline/loader/utils/libreoffice_converter.py`` is +reproduced below. + +Rendering goes through PyMuPDF (``fitz``), which is a declared backend +dependency, instead of Poppler. + +Usage: + python skills/bisheng-pptx/scripts/render_deck.py output/deck.pptx + python skills/bisheng-pptx/scripts/render_deck.py output/deck.pptx --pages 1-4 --width 1600 + +Prints the workspace-relative PNG paths; pass them to read_file to look at them. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +import traceback + +import fitz + +DEFAULT_WIDTH_PX = 1280 +SOFFICE_TIMEOUT_S = 300 + + +def _find_soffice() -> str | None: + for name in ("soffice", "libreoffice"): + found = shutil.which(name) + if found: + return found + return None + + +def _to_pdf(pptx_path: str, out_dir: str) -> str | None: + """Convert with a private LibreOffice profile; returns the pdf path or None.""" + soffice = _find_soffice() + if not soffice: + print("找不到 soffice/libreoffice，无法渲染预览。跳过视觉 QA，以 inspect_deck.py 的体检为准。") + return None + + abs_input = os.path.abspath(pptx_path) + os.makedirs(out_dir, exist_ok=True) + expected = os.path.join(out_dir, os.path.splitext(os.path.basename(abs_input))[0] + ".pdf") + + with tempfile.TemporaryDirectory() as profile: + command = [ + soffice, + "--headless", + "-env:SingleAppInstance=false", + f"-env:UserInstallation=file://{profile}", + "--convert-to", + "pdf", + "--outdir", + out_dir, + abs_input, + ] + try: + proc = subprocess.run(command, capture_output=True, text=True, timeout=SOFFICE_TIMEOUT_S) + except subprocess.TimeoutExpired: + print(f"soffice 转换超时（>{SOFFICE_TIMEOUT_S}s）。跳过视觉 QA。") + return None + + if not os.path.exists(expected): + # soffice exits 0 even when it converted nothing, so trust the file, not rc. + message = (proc.stdout + proc.stderr).strip() + print(f"soffice 未产出 PDF（rc={proc.returncode}）。输出: {message[:400]}") + if "could not be loaded" in message: + print("多半是这套环境的 LibreOffice 没装 Impress 组件（只有 Writer），无法打开 .pptx。") + print("跳过视觉 QA 即可，不影响 .pptx 交付；以 inspect_deck.py 的体检结果为准。") + return None + return expected + + +def _parse_pages(spec: str | None, total: int) -> list[int]: + if not spec: + return list(range(total)) + wanted: set[int] = set() + for chunk in spec.split(","): + chunk = chunk.strip() + if not chunk: + continue + if "-" in chunk: + start, end = chunk.split("-", 1) + wanted.update(range(int(start) - 1, int(end))) + else: + wanted.add(int(chunk) - 1) + return sorted(p for p in wanted if 0 <= p < total) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Render pptx slides to PNG for visual QA") + parser.add_argument("pptx", help="路径，相对工作区根，例如 output/deck.pptx") + parser.add_argument("--outdir", default=None, help="默认 scratch/preview/<文件名>/") + parser.add_argument("--pages", default=None, help="页码，如 1-4 或 2,5,7（默认全部）") + parser.add_argument("--width", type=int, default=DEFAULT_WIDTH_PX, help=f"输出宽度 px（默认 {DEFAULT_WIDTH_PX}）") + args = parser.parse_args() + + if not os.path.exists(args.pptx): + print(f"[FATAL] 文件不存在: {args.pptx}（请用相对工作区根的路径，例如 output/deck.pptx，不要带前导斜杠）") + return 0 + + # Always exit 0, including on a crash: a non-zero exit makes the code + # interpreter return stderr and drop stdout, hiding the reason. + try: + render(args.pptx, args.outdir, args.pages, args.width) + except Exception: + print("[FATAL] 渲染预览时出错：") + traceback.print_exc(file=sys.stdout) + print("视觉 QA 是可选步骤：跳过它，以 inspect_deck.py 的体检结果为准，不要因此重做 .pptx。") + return 0 + + +def render(pptx_path: str, outdir: str | None, pages: str | None, width_px: int) -> None: + stem = os.path.splitext(os.path.basename(pptx_path))[0] + out_dir = outdir or os.path.join("scratch", "preview", stem) + os.makedirs(out_dir, exist_ok=True) + + # Stale frames from a previous run read as "my fix did not apply". + for name in os.listdir(out_dir): + if name.startswith("slide-") and name.endswith(".png"): + os.remove(os.path.join(out_dir, name)) + + pdf_path = _to_pdf(pptx_path, out_dir) + if not pdf_path: + return + + written = [] + with fitz.open(pdf_path) as doc: + wanted = _parse_pages(pages, doc.page_count) + for index in wanted: + page = doc[index] + zoom = width_px / page.rect.width if page.rect.width else 1.0 + pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom)) + target = os.path.join(out_dir, f"slide-{index + 1}.png") + pix.save(target) + written.append(target) + + print(f"已渲染 {len(written)} 页（源 PDF: {pdf_path}）:") + for path in written: + print(f" {path}") + print("\n⚠️ 仅在你确知当前模型支持读图时才用 read_file 打开这些 PNG：它们会被编成 base64 图片块") + print(" 发给模型厂商，而默认的 Qwen/dashscope 通道已知不接收 base64 图片，读图可能直接失败甚至") + print(" 中断本次请求。拿不准就跳过，以 inspect_deck.py 的体检结果为准。") + print("看图时重点看：文字是否溢出/被截断、元素是否重叠、留白是否失衡。") + print("注意：渲染用的中文字体是文泉驿正黑，与用户 PowerPoint 里的实际字体宽度不同 ——") + print(" 预览里的文字松紧只作参考，容器请留约 10% 余量，不要为了预览效果反复微调字号。") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/SKILL.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/SKILL.md new file mode 100644 index 0000000000..39f6211ec2 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/SKILL.md @@ -0,0 +1,197 @@ +--- +name: bisheng-xlsx +description: >- + 制作或修改 Excel 表格（.xlsx）时使用：从零搭建数据表、明细表、汇总表、预算表、财务模型、 + 测算表、报价单、台账、看板底表；把整理好的数据导出成带公式和格式的工作簿；读取、清洗、 + 改写用户上传的 .xlsx/.xls/.csv。当用户说到「Excel」「表格」「工作簿」「xlsx」「报表」「台账」 + 「预算表」「测算」「明细表」「数据透视」「导出成表」，或提出「把这些数据做成 Excel」 + 「按这个模板填一张表」「帮我算一下并出个表」时触发。本技能给出 BiSheng 代码执行器里 + 唯一可行的 openpyxl 路径（该环境没有 markitdown，公式必须用 LibreOffice 重算才有值）， + 以及中文表格规范、审计配色、公式禁用清单、交付前的重算与体检脚本。 + 如果用户要的是图表图片、Word 文档或 PPT，不要用本技能。 +metadata: + display-name: Excel 表格制作（BiSheng 适配） +--- + +# 在 BiSheng 里做 XLSX + +## 0. 开工纪律 + +**这一轮只读文档，不要在同一轮里并行调用别的工具。** 读完本文件（必要时再读 references）之后， +下一轮才开始动手。把「读 SKILL.md」和「产出交付物」放进同一轮并行调用，等于技能没读。 + +**本技能要求已勾选代码执行器（`bisheng_code_interpreter`）。** 没有它就无法生成 .xlsx —— +直接告诉用户「请在工具里勾选代码执行器后重试」，不要用 Markdown 表格顶替。 + +## 1. 环境事实（照做，不要试探） + +| 项 | 事实 | +|---|---| +| **生成方式** | **只有 `openpyxl`**（后端 `pyproject.toml` 的正式依赖），`import openpyxl` 直接可用 | +| **公式重算** | 靠 LibreOffice（`soffice`）。**openpyxl 写出的公式没有值**，不重算等于交了张空表 | +| 不存在的东西 | `markitdown`、Node/npm、`defusedxml`、`pdftoppm`、`zip`/`unzip` 命令 | +| `xlsxwriter` | 在，但只能新建、不能读改已有文件，也不能和 openpyxl 混用 → **一律用 openpyxl** | +| 其它可用库 | pandas/numpy、Pillow、PyMuPDF(`fitz`)、matplotlib、python-docx、python-pptx、lxml | +| 禁止 | `pip install`、`npm install`、任何联网假设（生产多为离线内网） | +| 工作目录 | 执行器 cwd = 工作区根，**一律用相对路径** | +| `output/` | 唯一交付区。每次运行自动创建 | +| `scratch/` | 中间产物区，**不会交付**，需自己 `os.makedirs`；**跨轮追问时会丢** | +| `uploads/` | 用户上传的原件在这里。跨轮追问时和 `output/` 一起被带进新会话 | +| `skills/bisheng-xlsx/` | 本技能包，脚本和参考资料在这里，只读 | +| 绝对禁止 | 写 `/output/xxx.xlsx` 这种带前导斜杠的路径 —— 文件被静默丢弃，用户拿不到 | +| 单次执行上限 | 本地执行器 600 秒；**E2B 沙箱 300 秒**。构建、重算、体检分多次调用 | +| 日志规则 | 二选一：`returncode != 0` **只回 stderr**（stdout 整段丢弃）；`returncode == 0` **只回 stdout**（stderr 整段丢弃）→ 一切诊断用 `print()`，且子进程的 stdout 和 stderr 都要各自打印一遍 | +| 可见性 | `exitcode 0` 时执行器把本轮新建/修改的文件同步进工作区，之后 `ls`/`read_file` 一般能看到。但**判断是否产出以 `exitcode 0` + 日志为准**，不要因为一次 `ls` 没看到就重做 | +| 轮次 | 最后两轮工具收窄到 `write_file`/`edit_file`/导出，代码执行器会被拦掉 → 表要尽早产出，不要拖到收尾 | +| **E2B 沙箱** | 沙箱模式下 `skills/` **结构性不可见**（copy-in 快照早于技能物化）→ 调包内脚本必 `FileNotFoundError`。一旦报这个错，就改走纯 openpyxl 内联写法：不调本包脚本，公式改成 Python 算好写数值，并在收尾里说明没做重算与体检 | + +### 1.1 先探一次环境（第一次执行代码时顺手做，只花一轮） + +```python +import shutil +try: + import openpyxl + print("openpyxl OK", openpyxl.__version__) +except ImportError: + print("openpyxl MISSING") +print("soffice:", shutil.which("soffice") or shutil.which("libreoffice") or "无 —— 公式无法重算，见 §4 降级方案") +``` + +- `openpyxl MISSING`：正常部署不会出现。**不要 `pip install`**（共享的离线环境，装了会污染所有租户）。 + 如实告诉用户环境缺依赖，需要运维补装。 +- `soffice` 没有、或重算时报「没有 Calc 组件」：见 §4 的降级方案，**不要假装重算过了**。 + +## 2. 选路线 + +| 情况 | 做法 | +|---|---| +| 用户给了数据（或让你先查再整理），要一张新表 | §3 从零构建 | +| 用户上传了 .xlsx 要改内容 / 填数 | §5 改已有文件（**先读懂它的约定，再动手**） | +| 用户上传了 .csv/.xls 要清洗成规范表 | pandas 读进来清洗 → 按 §3 写出 | +| 用户要的是图表图片、Word、PPT | 不属于本技能 | + +## 3. 从零构建 + +**第 1 步 · 定结构**。先想清楚分几个表、每个表的列。复杂测算把假设单独放一个表。 +草稿写 `scratch/`，**不要写进 `output/`** —— 交付物按类型排序，`.md`/`.docx`/`.pdf`/`.html` 排在 +`.xlsx` 前面，所以 `output/` 里只要有一个 `.md`，它就**永远**压过你的表成为用户看到的头条交付物。 +不是「同一轮才会」，是只要在就会。 + +**第 2 步 · 写构建脚本**。用 `write_file` 把完整脚本写到 `scratch/build_sheet.py`， +**不要把整段代码塞进代码执行器的参数里** —— 参数过长会被截断，导致反复重试却总差一截。 +写法读 `/skills/bisheng-xlsx/references/openpyxl-cookbook.md`（表格、样式、数字格式、公式、 +图表、批注都有可直接抄的片段），规范读 `/skills/bisheng-xlsx/references/design-zh.md`。 + +**第 3 步 · 执行**： + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "scratch/build_sheet.py"], capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +> 为什么不直接写 `python scratch/build_sheet.py`：PATH 里的 `python` 未必是后端那个解释器， +> 用 `sys.executable` 才能保证跑在装了 openpyxl 的环境里。**下面所有脚本调用都用这个写法。** + +**第 4 步 · 重算（有公式就必做）**，见 §4。 +**第 5 步 · 体检并返修**，见 §6。返修用 `edit_file` 定点改 `scratch/build_sheet.py` 再重跑， +不要每次重写整份脚本。 + +## 4. 重算公式（有公式就必做） + +openpyxl 写出的公式**只是字符串，没有结果**。不重算的话，pandas、`data_only=True`、 +以及多数预览器读到的全是空 —— 用户打开看到的是一张有公式没数字的表。 + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-xlsx/scripts/recalc_check.py", "output/xxx.xlsx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +LibreOffice 会算完所有公式、**就地重写文件**，然后脚本回读并列出所有错误单元格。 +一次重算连冷启动约 10–30 秒；超时就加 `--timeout 120`。 +看结论： + +- `重算完成` —— 公式都能算出结果。**这不是交付条件**，还要过 §6 的体检。 +- `重算未通过` + 错误单元格清单 —— 逐个改完再重算。 +- `[未重算]` —— 环境问题（没有 soffice、或只装了 writer 没装 calc、或超时）。**这时不要硬撑**： + 改成在 Python 里把数算好直接写数值，并在表里用一列文字说明计算口径， + 同时如实告诉用户「本环境无法重算公式，已改为写入计算结果，修改输入不会自动重算」。 + +> **能算 ≠ 算对。** 区间差一行、引用错行，照样是一张干净的错数字表。 +> 铺开整张表之前，先写 2–3 个关键公式，重算一次，肉眼核对结果符不符合预期。 + +## 5. 改用户上传的表 + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-xlsx/scripts/inspect_workbook.py", + "uploads/原表.xlsx", "--content-only"], capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +它按单元格坐标打印内容和公式（这是 `markitdown` 在本环境的替代品，而且比它多给坐标， +所以可以据此规划改哪一格）。然后： + +- **原表的约定压倒本技能的一切规范** —— 它用什么字体、什么数字格式、什么配色，就跟着它。 +- **先找到它的输入格**（通常有独特的字色或填充），只在那里写值，**不要动任何已有公式**。 +- 用默认方式 `load_workbook(path)` 打开（**不要加 `data_only=True`**，那样保存会把公式全变成字面量）。 +- `.xlsm` 要 `keep_vba=True`，否则宏全丢。 +- 改完另存到 `output/`，不要覆盖 `uploads/` 里的原件。 +- **不要给别人的表加示例行。** + +## 6. 交付前体检（必做） + +```python +import subprocess, sys +r = subprocess.run([sys.executable, "skills/bisheng-xlsx/scripts/inspect_workbook.py", "output/xxx.xlsx"], + capture_output=True, text=True) +print(r.stdout or "(no stdout)") +print(r.stderr[-2000:] if r.stderr else "(no stderr)") +``` + +输出分两段： + +- **内容**：逐表逐格打印，公式旁边跟着重算后的值。用它核对数字对不对、有没有缺漏。 +- **体检**：ERROR 必须修完再交付；WARN 逐条复核；INFO 是建议。覆盖：禁用函数（含 `_xlfn.` 前缀伪装）、 + `_xlfn.` 缺前缀、跨表引用引号、百分比存成文本、百分比存整数、数字带单位存成文本、 + 公式无缓存值、重算后仍是错误值、缺表头、列宽不足导致 `###`、合并区丢值、年份千分位、 + 残留占位符、硬编码系数、除法未保护。 + 同一条规则在同一列命中多次时只列 3 例 + 一句「同类还有 N 处」，按同样办法一次改完。 + +改完重新生成 → 重算 → 再体检，直到最后一行是 `结论: 通过`。 +只有 ERROR 挡交付；`结论: 通过` 之后仍要把 WARN 逐条看一遍，确实无碍再交付。 + +## 7. 公式红线（体检会拦，但你一开始就别写） + +- ❌ **`XLOOKUP` / `XMATCH` / `SORT` / `FILTER` / `UNIQUE` / `SEQUENCE` / `LET` / `LAMBDA`** + —— openpyxl 写不出溢出元数据，LibreOffice 要么报 `#NAME?`，要么只填左上角一个值 + **而且重算报 0 错误**。查表用 `INDEX`/`MATCH`；排序、去重、筛选在 Python 里做完再写值。 + **加 `_xlfn.` 前缀救不了这一组**：`=_xlfn.SORT(...)` 实测照样 `#NAME?`，体检也照样拦。 +- ⚠️ **`TEXTJOIN` / `CONCAT` / `IFS` / `SWITCH` / `MAXIFS` / `MINIFS` 必须带 `_xlfn.` 前缀**， + 裸写 → `#NAME?`。 +- ⚠️ 表名含空格的跨表引用必须加单引号：`='假设 输入'!$B$5`。用 `sheet_ref()` 自动处理。 +- ⚠️ 百分比存小数（0.15 = 15%）。用 `set_pct()`，它会在你传 15 时直接报错。 + +## 8. 交付纪律 + +- `output/` 里**只放最终的 `.xlsx`**。草稿、构建脚本、中间版本一律放 `scratch/`。 +- 文件名用有意义的中文名，如 `output/2024年度经营分析.xlsx`。 +- 收尾时如实说明做了什么、几个表、多少行、公式是否已重算。 + **不要声称生成了实际不存在的文件。** +- 如果因为环境限制改成了写死数值，必须在收尾里明确说出来。 + +## 9. 绝不要做的事 + +- ❌ 跑 `markitdown` 读表 —— 环境里没有，用 §5 的 `--content-only`。 +- ❌ `pip install` 任何东西。 +- ❌ 用绝对路径 `/output/...`。 +- ❌ 有公式却不重算就交付。 +- ❌ 把表降级成 Markdown 表格或 CSV 交付。用户要的是 .xlsx。 +- ❌ `load_workbook(..., data_only=True)` 之后 `save()` —— 会永久毁掉所有公式。 +- ❌ 因为一次 `ls` 没看到刚生成的文件就重做一遍 —— `exitcode 0` + 日志就是产出的凭据。 +- ❌ 把百分比、金额写成 `"15%"`、`"1,250,000"` 这种字符串 —— 不参与求和，体检会拦。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/references/design-zh.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/references/design-zh.md new file mode 100644 index 0000000000..bf2a0b8d9b --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/references/design-zh.md @@ -0,0 +1,88 @@ +# 中文表格设计规范 + +给「交给别人看、别人还要在上面改」的表用。核心不是好看，是**别人能看懂你怎么算的**。 + +## 1. 结构 + +- **一个表只做一件事**：`假设` / `明细` / `汇总` / `图表` 分表放，不要把假设埋在明细中间。 +- **首行是表头，第二行起是数据**。不要在表头上面塞标题行又不合并，那会让筛选和透视全乱。 + 真要大标题，合并 `A1:xx1` 单独一行，数据表头从第 3 行开始，并 `freeze_and_filter(ws, header_row=3)`。 +- **列顺序**：维度列（名称/分类/时间）在左，数值列在右，同比/占比这类派生列最后。 +- **不要有隐藏行列**，也不要用空行分组 —— 用小计行 + 加粗代替。 +- 表名和列名用**用户说的词**。他叫「营收」你就别写「主营业务收入」。 + +## 2. 审计配色（财务/预算类必守） + +这是财务圈通用约定，评审的人先看颜色再看数字： + +| 颜色 | 含义 | helper | +|---|---|---| +| **蓝字** `0000FF` | 人手填的硬编码输入、情景假设 | `mark_input(cell)` | +| **黑字** `000000` | 公式算出来的 | `mark_formula(cell)` | +| **绿字** `008000` | 引用了本工作簿其它表 | `mark_cross_sheet(cell)` | +| **红字** `FF0000` | 引用了另一个文件 | — | +| **黄底** `FFFF00` | 关键假设、需要用户填的格子 | `mark_assumption(cell)` | + +**每一个硬编码数字都要能追溯来源**：写进单元格批注（`add_note`），或在表格末尾相邻单元格注明。 +有出处就写出处（`来源：2024 年报 P32 收入附注`），是用户给的就写「用户提供」。 +没有来源的数字不要出现在交付物里。 + +## 3. 数字格式 + +| 场景 | 格式 | 说明 | +|---|---|---| +| 金额 | `#,##0;(#,##0);-` | 负数括号、0 显示为 `-`；单位写在列名里（`收入（万元）`） | +| 金额（保留一位） | `#,##0.0;(#,##0.0);-` | 以「万元/百万」为单位时用 | +| 百分比 | `0.0%` | **值存小数**，0.15 → 15.0% | +| 倍数 | `0.0x` | 估值倍数 | +| 年份 | `@`（文本） | 否则会变成 `2,024` | +| 日期 | `yyyy-mm-dd` | 不要用 `2024/1/1` 这种依赖区域设置的写法 | + +- 同一列格式必须统一，不要有的带千分位有的不带。 +- 单位统一写在**列名**里，不要在每个单元格后面拼「元」——那会让数字变成文本，没法算。 +- **千分位和百分号是格式，不是内容**。写 `1250000` + `number_format="#,##0"`， + 不要写字符串 `"1,250,000"`；写 `0.15` + `0.0%`，不要写 `"15%"`。 + 存成文本的数字选中整列不显示求和，排序按字符串排，体检会拦。 + +## 4. 公式纪律 + +- **有公式就别写死结果**。`=SUM(B2:B9)`，不要在 Python 里算好再写 `12500000`。 + 表的价值在于改了输入会重算；写死了就只是张图片。 +- **假设放在有标签的单元格里，公式引用它**：`=B5*(1+$B$6)`，不要 `=B5*1.05`。 +- **同一行的公式必须一致**。整行里某一格被单独改过，是最常见也最难发现的错。 +- **分母可能为 0 的地方加 `IFERROR`**。 +- 写完先抽查 2–3 个关键公式的结果对不对，再铺开整张表。 + **重算通过只证明公式「能算」，不证明「算得对」** —— 区间差一行照样是一张干净的错表。 + +## 5. 列宽与可读性 + +- 中文占两个半角宽，`autofit_columns()` 已按双宽计算。手工设宽时也要按这个算， + 否则数字列会显示成 `###`（这是最常被用户投诉的"表坏了"）。 +- 超过 15 行就冻结表头（`freeze_and_filter`）。 +- 长文本列设 `wrap_text=True` 并调行高，不要让它横着撑到 80 字符宽。 +- 字体用**微软雅黑**或**等线**（用户机器一定有）。表头 11pt 加粗，正文 10pt。 + +## 6. 给别人填的表 + +- 顶部或侧边放一小段**填写说明**（`add_legend`）：哪些格子要填、填什么格式。 +- 给一行**示例数据**，用真实感的值，并标明"示例行，填写前请删除"。 +- 需要填的格子用黄底标出来，并加数据验证（下拉、数值范围），减少填错。 +- **改别人的表时绝不要加示例行**，也不要动已有公式 —— 先找到它的输入格 + （通常有独特的字色或填充），只在那里写。 + +## 7. 交付前 + +按顺序跑完这两步，两个都过了才算完： + +1. `recalc_check.py` —— 公式必须有缓存值且无错误单元格。 +2. `inspect_workbook.py` —— ERROR 清零，WARN 逐条复核。 + +## 8. 常见"一眼就知道是机器做的" + +- 所有列等宽，中文列被挤成 `###`。 +- 百分比显示成 `1500.0%`（存了 15 而不是 0.15）。 +- 年份显示成 `2,024`。 +- 合计行没有加粗，也没有和明细拉开距离。 +- 通篇没有一个批注，没有一处写来源。 +- 表名还是 `Sheet1`。 +- 数字后面拼了单位变成文本，选中一列右下角不显示求和。 diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/references/openpyxl-cookbook.md b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/references/openpyxl-cookbook.md new file mode 100644 index 0000000000..3ce5a69861 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/references/openpyxl-cookbook.md @@ -0,0 +1,247 @@ +# openpyxl 速查（BiSheng 适配） + +可直接抄的片段。所有代码假定在**构建脚本**里（`scratch/build_sheet.py`），由代码执行器用 +`subprocess.run([sys.executable, "scratch/build_sheet.py"])` 跑。 + +开头统一这样起： + +```python +import sys +sys.path.insert(0, "skills/bisheng-xlsx/scripts") # 本技能包的 helper +from xlsx_helpers import ( + write_table, autofit_columns, style_header, freeze_and_filter, + mark_input, mark_formula, mark_assumption, add_note, set_pct, sheet_ref, + add_legend, FMT, FONT_NAME, +) +from openpyxl import Workbook, load_workbook +import os +os.makedirs("output", exist_ok=True) +os.makedirs("scratch", exist_ok=True) +``` + +--- + +## 1. 新建 / 打开 / 保存 + +```python +wb = Workbook() +ws = wb.active +ws.title = "利润预测" # 表名 ≤31 字符，不能含 : \ / ? * [ ] +ws2 = wb.create_sheet("假设") # 追加 +ws3 = wb.create_sheet("封面", 0) # 插到最前 + +wb.save("output/2024年度经营分析.xlsx") +``` + +打开已有文件 —— **读一个模型要加载两次**，一次拿公式一次拿值，一次加载给不了两者： + +```python +wb_f = load_workbook("uploads/原表.xlsx") # 公式字符串，无值 +wb_v = load_workbook("uploads/原表.xlsx", data_only=True) # 缓存值，无公式 +``` + +> ⚠️ **`data_only=True` 打开后再 `save()` 会永久毁掉公式** —— 保存下去的是一堆字面量。 +> 要改文件就用默认方式打开。 +> ⚠️ `.xlsm` 必须 `load_workbook(path, keep_vba=True)`，否则宏全丢。 + +## 2. 写单元格 + +```python +ws["A1"] = "项目" +ws.cell(row=2, column=1, value="主营业务收入") # 行列号从 1 开始 +ws["B2"] = 12500000 +ws["B3"] = "=SUM(B4:B9)" # 以 = 开头即公式 +``` + +批量写一行： + +```python +for r, (name, amount) in enumerate(data, start=2): + ws.cell(row=r, column=1, value=name) + ws.cell(row=r, column=2, value=amount).number_format = FMT["money"] +``` + +## 3. 数字格式（最容易出错的地方） + +```python +cell.number_format = FMT["money"] # #,##0;(#,##0);- 负数括号、0 显示为 - +cell.number_format = FMT["pct"] # 0.0% +cell.number_format = FMT["mult"] # 0.0x +cell.number_format = FMT["date"] # yyyy-mm-dd +cell.number_format = "@" # 文本（年份用它，避免 2,024） +``` + +**百分比必须存小数**：15% 存 `0.15`，存 `15` 会显示成 `1500.0%`。 +用 `set_pct(cell, 0.15)`，它会在你传 15 时直接报错而不是默默出个错 100 倍的表。 + +```python +set_pct(ws["C2"], 0.152) # 显示 15.2% +ws["E2"] = "2024" # 年份写成字符串，或 number_format="@" +``` + +## 4. 表格（一次搞定表头 + 数据 + 格式 + 边框） + +```python +first, last = write_table( + ws, + headers=["项目", "2024年", "2025E", "增长率"], + rows=[["营业收入", 12500000, 14200000, 0.136], + ["营业成本", 8200000, 9100000, 0.110]], + start_row=3, + number_format={1: "money", 2: "money", 3: "pct"}, # 键是**列偏移**，从 0 数 +) +ws.cell(row=last + 1, column=1, value="合计") +ws.cell(row=last + 1, column=2, value=f"=SUM(B{first}:B{last})").number_format = FMT["money"] + +autofit_columns(ws) # 中文按双宽算，避免 ### +freeze_and_filter(ws, header_row=3) +``` + +## 5. 样式 + +```python +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side + +cell.font = Font(name="微软雅黑", size=11, bold=True, color="FFFFFF") +cell.fill = PatternFill("solid", fgColor="1F4E79") +cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) + +thin = Side(style="thin", color="BFBFBF") +cell.border = Border(left=thin, right=thin, top=thin, bottom=thin) + +ws.column_dimensions["A"].width = 24 # 单位≈半角字符数 +ws.row_dimensions[1].height = 22 +``` + +列宽按**渲染后**的宽度算，不是按存进去的值：`12500000` 配 `#,##0` 画出来是 `12,500,000`， +11 个半角宽；中文再翻倍。体检脚本就是这么量的。公式的结果宽度**重算后才知道**， +所以顺序是：写公式 → `recalc_check.py` 重算 → **再跑一次 `autofit_columns(ws)`** → 保存 → 体检。 + +**审计配色**（财务模型必守，见 design-zh.md §2）： + +```python +mark_input(ws["B2"], note="来源：2024 年报 P32") # 蓝字 = 人填的硬编码 + 批注写来源 +mark_formula(ws["B3"]) # 黑字 = 公式 +mark_assumption(ws["B6"], note="管理层给定") # 黄底蓝字 = 关键假设/待用户填 +``` + +## 6. 合并单元格 + +```python +ws.merge_cells("A1:D1") +ws["A1"] = "2024 年度经营分析" # 只写左上角！ +``` + +> 合并区内**除左上角外都是 `MergedCell`，`.value` 只读**，写进去会丢。先写值再合并也一样。 + +## 7. 公式：哪些能用，哪些是坑 + +| 类别 | 结论 | +|---|---| +| `SUM` `SUMIFS` `INDEX` `MATCH` `IFERROR` `SUMPRODUCT` `VLOOKUP` | ✅ 直接用 | +| `TEXTJOIN` `CONCAT` `IFS` `SWITCH` `MAXIFS` `MINIFS` | ⚠️ **必须写 `_xlfn.` 前缀**，如 `=_xlfn.TEXTJOIN(",",TRUE,A2:A9)`。裸写 → `#NAME?` | +| `XLOOKUP` `XMATCH` `SORT` `FILTER` `UNIQUE` `SEQUENCE` `LET` `LAMBDA` | ❌ **绝对不要用**。见下 | +| 数组公式 / 动态数组 | ❌ openpyxl 写不出溢出元数据 | + +**为什么 XLOOKUP 那一组是硬禁**：openpyxl 写出的文件没有溢出（spill）元数据， +LibreOffice 要么算不出（`#NAME?`），要么只给左上角一个值 —— 后一种情况**重算脚本会报 +`total_errors: 0`**，用户打开才发现半张表是空的。查表一律用 `INDEX`/`MATCH`； +排序、去重、筛选**在 Python 里做完再写值**。 + +`_xlfn.` 前缀只对上面那一行「⚠️」的六个函数有用，**对硬禁那一组无效** —— +实测 `=_xlfn.SORT(A1:A9)` 重算后就是 `#NAME?`，体检也会当成禁用函数拦下来。 + +跨表引用用 helper，它会按需加引号： + +```python +ws["B2"] = sheet_ref("假设 输入", "$B$5") # → ='假设 输入'!$B$5 +``` + +> 实测：`=假设 输入!$B$2`（表名含空格不加引号）重算成 `#VALUE!`； +> `=关键假设!$B$2`（纯中文表名不加引号）正常出值。**纯中文表名不用加引号**，加了也无害。 + +保护除法： + +```python +ws["D2"] = "=IFERROR(B2/C2,0)" +``` + +## 8. 批注（假设的来源写这里） + +```python +add_note(ws["B6"], "假设：2025 年收入增速 13.6%\n来源：管理层 2025 预算，2025-01 版") +``` + +## 9. 图表（原生图表，用户可以改数据） + +```python +from openpyxl.chart import BarChart, LineChart, Reference + +chart = BarChart() +chart.title = "分季度收入" +chart.y_axis.title = "金额（元）" +data = Reference(ws, min_col=2, min_row=3, max_row=last) # 含表头行则 titles_from_data=True +cats = Reference(ws, min_col=1, min_row=4, max_row=last) +chart.add_data(data, titles_from_data=True) +chart.set_categories(cats) +chart.width, chart.height = 18, 9 # 厘米 +ws.add_chart(chart, "F3") +``` + +> 原生图表比贴 matplotlib 图片好：用户改了数字图会跟着变。只有需要复杂可视化时才退回图片。 + +## 10. 插入图片 + +```python +from openpyxl.drawing.image import Image +img = Image("scratch/trend.png") +img.width, img.height = 480, 270 # 像素 +ws.add_image(img, "H3") +``` + +> 图片文件在保存前不能删；`Image` 是懒加载的。 + +## 11. 条件格式与数据验证 + +```python +from openpyxl.formatting.rule import CellIsRule, ColorScaleRule +from openpyxl.worksheet.datavalidation import DataValidation + +ws.conditional_formatting.add( + "D4:D20", CellIsRule(operator="lessThan", formula=["0"], font=Font(color="FF0000"))) +ws.conditional_formatting.add( + "B4:B20", ColorScaleRule(start_type="min", start_color="FFFFFF", + end_type="max", end_color="63BE7B")) + +dv = DataValidation(type="list", formula1='"是,否"', allow_blank=True) +ws.add_data_validation(dv) +dv.add("E4:E20") +``` + +## 12. 页面设置（要打印时才需要） + +```python +ws.page_setup.orientation = "landscape" +ws.page_setup.fitToWidth = 1 +ws.sheet_properties.pageSetUpPr.fitToPage = True +ws.print_title_rows = "1:3" # 每页重复表头 +``` + +## 13. 读已有表做分析 + +```python +wb = load_workbook("uploads/数据.xlsx", data_only=True, read_only=True) +ws = wb["Sheet1"] +rows = list(ws.iter_rows(min_row=2, values_only=True)) +wb.close() +``` + +> `read_only=True` 读大文件省内存，但拿不到样式，也不能写。 + +pandas 也在，批量数据进出更省事： + +```python +import pandas as pd +df = pd.read_excel("uploads/数据.xlsx", sheet_name="明细") +df.to_excel("output/清洗后.xlsx", index=False) # 但样式要用 openpyxl 再补 +``` diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/inspect_workbook.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/inspect_workbook.py new file mode 100644 index 0000000000..0298882aa8 --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/inspect_workbook.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Read out and health-check an .xlsx, for the BiSheng code interpreter. + + python skills/bisheng-xlsx/scripts/inspect_workbook.py output/x.xlsx + python skills/bisheng-xlsx/scripts/inspect_workbook.py output/x.xlsx --content-only + python skills/bisheng-xlsx/scripts/inspect_workbook.py output/x.xlsx --check-only + +Two jobs the official skill delegates to things this environment does not have: + +* **Content read-out** replaces ``markitdown`` (absent here), and unlike it this + prints cell coordinates, so edits can be planned from the output. +* **Health check** replaces "the model should remember ~10 negative rules". + Every rule the official SKILL.md states as prose and never verifies — banned + spilling functions, the ``_xlfn.`` prefix set, percentages stored as text, + unquoted sheet names, CJK column width — is checked mechanically here. + +Always exits 0: the BiSheng executor discards stdout on a non-zero exit, which +would throw away the entire report. +""" + +import argparse +import datetime as dt +import os +import re +import sys +import traceback + +try: + from openpyxl import load_workbook + from openpyxl.utils import get_column_letter, range_boundaries +except ImportError: # pragma: no cover - only when the image is cut down + print("[FATAL] openpyxl 不可用；无法体检。请让运维在后端环境确认依赖。") + sys.exit(0) + +# LibreOffice cannot evaluate these under any prefix, and — worse — on builds +# that do evaluate them, an openpyxl-written file carries no spill metadata, so +# only the top-left cell gets a value and the recalc reports zero errors. +BANNED = ["XLOOKUP", "XMATCH", "SORT", "FILTER", "UNIQUE", "SEQUENCE", "LET", "LAMBDA", "TEXTSPLIT"] + +# Post-2007 names Excel stores prefixed; openpyxl writes the string verbatim, so +# a bare name becomes a literal #NAME? in the delivered file. +NEEDS_XLFN = ["TEXTJOIN", "CONCAT", "IFS", "SWITCH", "MAXIFS", "MINIFS"] + +# A banned name stays banned behind the `_xlfn.` prefix — writing `_xlfn.SORT` +# does not make LibreOffice able to spill. So the prefix must be part of the +# match, not something the leading guard silently swallows: with a bare +# `(? None: + """Record a finding, dropping exact repeats and tagging it for collapsing. + + Two things ruin this report as an input to a model. Exact repeats: a formula + like ``=SUM(区域!A:A)+区域!B1`` used to emit the same sentence twice. And one + rule firing per cell: a 500-row column with no number format used to print + 500 identical INFO lines and bury every ERROR above them. ``group`` names the + rule+scope so the printer shows a few examples and a count. + """ + key = (level, message) + if key in _seen: + return + _seen.add(key) + findings.append((level, message, group or message)) + + +def cjk_width(text) -> int: + """Display width in half-widths. Must stay identical to + ``xlsx_helpers.cjk_width`` — the helper sizes the columns, this measures + them, and two different definitions make the checker fight the builder.""" + if text is None: + return 0 + width = 0 + for ch in str(text): + # CJK ideographs, CJK punctuation, kana, and full-width forms. + if "一" <= ch <= "鿿" or "　" <= ch <= "〿" or "぀" <= ch <= "ヿ" or "＀" <= ch <= "￯": + width += 2 + else: + width += 1 + return width + + +def rendered_text(cell, cached=None) -> str: + """What Excel actually draws in the cell, honouring ``number_format``. + + Measuring the raw value under-counts by exactly the characters that cause + ###: 12500000 is 8 wide as a Python int but ``#,##0`` renders it as + ``12,500,000``, 10 wide. ``cached`` supplies the recalculated value for a + formula cell, whose own text (``=SUM(B2:B9)``) says nothing about its width. + """ + value = cell.value + if isinstance(value, str) and value.startswith("="): + value = cached + if value is None: + return "" + fmt = (cell.number_format or "General").split(";")[0] + if isinstance(value, (dt.datetime, dt.date, dt.time)): + return "X" * min(20, max(8, len(fmt))) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return str(value) + + if fmt in ("General", "@") or not any(ch in fmt for ch in "0#"): + # General has no fixed rendering: Excel rounds the display down to + # whatever fits, so measuring it as `repr` over-states the need. Cap it + # at the width beyond which Excel starts shortening on its own. + return str(value)[:11] + + scaled = value * 100 if "%" in fmt else value + match = DECIMALS_RE.search(fmt) + decimals = len(match.group(1)) if match else 0 + grouped = "#,##" in fmt or "0,00" in fmt + body = f"{abs(scaled):,.{decimals}f}" if grouped else f"{abs(scaled):.{decimals}f}" + if scaled < 0: + body = "-" + body + for symbol in ("¥", "￥", "$", "€"): + if symbol in fmt: + body = symbol + body + break + if "%" in fmt: + body += "%" + return body + + +def needs_quoting(sheet_name: str) -> bool: + """Whether a cross-sheet reference to this name *must* be single-quoted. + + Deliberately narrower than the quoting rule in ``xlsx_helpers.sheet_ref``: + that one quotes liberally because quoting is always safe, while flagging + something as an ERROR needs certainty. A pure-CJK name like ``利润预测`` is + perfectly legal unquoted (verified: LibreOffice evaluates ``=假设!$B$2``), + so treating "non-ASCII" as "must quote" would false-positive on nearly every + Chinese workbook. + """ + return any(ch in sheet_name for ch in " -()&'+.") or (sheet_name[:1].isdigit() if sheet_name else False) + + +def hardcoded_coefficients(formula: str) -> list[str]: + """Bare multipliers/divisors that are neither unit conversions nor 1-digit.""" + hits = [] + for match in COEFFICIENT_RE.finditer(formula): + number = match.group(1) + if formula[match.end() : match.end() + 1] in {"%", "E", "e"}: + continue + if number in UNIT_CONSTANTS: + continue + if "." not in number and len(number) < 2: + continue + hits.append(number) + return hits + + +def fmt_value(value) -> str: + if value is None: + return "" + text = str(value).replace("\n", "⏎") + return text if len(text) <= 60 else text[:57] + "…" + + +# --------------------------------------------------------------------------- # +# Content read-out +# --------------------------------------------------------------------------- # + + +def dump_content(path: str, max_rows: int) -> None: + wb = load_workbook(path, data_only=False) + wb_values = None + try: + wb_values = load_workbook(path, data_only=True) + except Exception: + pass # cached values are a bonus here; the formula pass is the required one + + print("=" * 72) + print(f"内容 · {os.path.basename(path)} · {len(wb.sheetnames)} 个工作表") + print("=" * 72) + + for name in wb.sheetnames: + ws = wb[name] + vs = wb_values[name] if wb_values is not None and name in wb_values.sheetnames else None + print(f"\n## 工作表「{name}」 {ws.max_row} 行 × {ws.max_column} 列", end="") + if ws.freeze_panes: + print(f" 冻结={ws.freeze_panes}", end="") + if ws.auto_filter and ws.auto_filter.ref: + print(f" 筛选={ws.auto_filter.ref}", end="") + merged = list(ws.merged_cells.ranges) + if merged: + print(f" 合并={len(merged)}处", end="") + print() + + shown = 0 + for row in ws.iter_rows(): + if shown >= max_rows: + print(f" … 其余 {ws.max_row - shown} 行未显示（--max-rows 可调）") + break + cells = [c for c in row if c.value is not None] + if not cells: + continue + parts = [] + for cell in cells: + text = fmt_value(cell.value) + if isinstance(cell.value, str) and cell.value.startswith("="): + cached = vs[cell.coordinate].value if vs is not None else None + text = f"{text} →{fmt_value(cached) if cached is not None else '未重算'}" + parts.append(f"{cell.coordinate}={text}") + print(" " + " | ".join(parts)) + shown += 1 + wb.close() + if wb_values is not None: + wb_values.close() + + +# --------------------------------------------------------------------------- # +# Health checks +# --------------------------------------------------------------------------- # + + +def check_formulas(ws, name: str, sheet_names: set[str]) -> int: + formula_count = 0 + for row in ws.iter_rows(): + for cell in row: + value = cell.value + if not isinstance(value, str): + continue + where = f"{name}!{cell.coordinate}" + stripped = value.strip() + + # Exact match only. `=IF(ISNA(X),"#N/A",X)` is a legitimate formula; + # a baked error is a cell whose entire content IS the literal. + if stripped in ERROR_LITERALS: + add( + "ERROR", + f"{where} 单元格里是错误值 {stripped}（不是公式，是已经算坏的结果）", + f"{name} 错误值字面量", + ) + continue + + if any(p in value for p in PLACEHOLDERS): + add("WARN", f"{where} 残留占位文本：{fmt_value(value)}", f"{name} 残留占位文本") + + if not value.startswith("="): + if TEXT_PCT_RE.match(stripped): + add( + "ERROR", + f"{where} 百分比存成了文本 {stripped!r} → 不参与求和、排序按字符串排。" + f"改成写数值 {stripped.rstrip('%').strip()} / 100 并设 number_format='0.0%'", + f"{name} 百分比存成文本", + ) + elif TEXT_NUM_RE.match(stripped) or TEXT_GROUPED_RE.match(stripped): + add( + "WARN", + f"{where} 数字存成了文本 {stripped!r}（带了单位或千分位）→ 选中整列右下角不显示求和。" + f"单位写进列名，单元格只放数值，千分位交给 number_format='#,##0'", + f"{name} 数字存成文本", + ) + continue + + formula_count += 1 + upper = value.upper() + + for fn, pattern in BANNED_RE.items(): + if pattern.search(upper): + add( + "ERROR", + f"{where} 用了 {fn}()——LibreOffice 重算不了，交付出去在用户 Excel 里可能是 #NAME?，" + f"且重算脚本抓不到。改用 INDEX/MATCH，或在 Python 里排序去重后直接写值。", + f"{name} 用了 {fn}", + ) + for fn, pattern in XLFN_RE.items(): + if pattern.search(upper): + add( + "ERROR", + f"{where} {fn}() 缺 `_xlfn.` 前缀 → 会变成字面量 #NAME?。写成 `_xlfn.{fn}(...)`", + f"{name} {fn} 缺前缀", + ) + + # Drive this off the real sheet names rather than trying to infer a + # name from the bare match: `=假设 输入!$B$2` tokenises as the bare + # name `输入`, so the space is simply not visible from the match. + for sname in sheet_names: + if not needs_quoting(sname): + continue + if f"{sname}!" in value and f"'{sname}'!" not in value: + add( + "ERROR", + f"{where} 跨表引用「{sname}」名字含空格/特殊字符却没加单引号 → #VALUE!。写成 '{sname}'!", + f"{name} 引用「{sname}」缺单引号", + ) + + for match in SHEET_REF_RE.finditer(value): + inner, bare = match.group(1), match.group(2) + ref_name = inner or bare + if not ref_name or ref_name in sheet_names: + continue + if ref_name.upper() in {"TRUE", "FALSE"}: + continue + # A bare token before `!` is only a sheet reference if it is not + # the tail of a quoted name we already matched. + if any(ref_name in s for s in sheet_names): + continue + add( + "WARN", + f"{where} 引用了工作表「{ref_name}」，但工作簿里没有这个表名", + f"{name} 引用不存在的表 {ref_name}", + ) + + coefficients = hardcoded_coefficients(value) + if coefficients: + add( + "WARN", + f"{where} 公式里硬编码了系数 {'、'.join(coefficients)}：{fmt_value(value)} " + f"→ 把它挪到有标签的假设单元格并引用它", + f"{name} 硬编码系数", + ) + + if "IFERROR" not in upper and re.search(r"/\s*(?:'[^']+'!|[\w一-鿿]+!)?\$?[A-Za-z]{1,3}\$?\d", value): + add("INFO", f"{where} 除法未加 IFERROR 保护：{fmt_value(value)}", f"{name} 除法未加 IFERROR") + return formula_count + + +def check_header(ws, name: str) -> None: + """First data-ish row must be column names, not the first data record.""" + if ws.max_column < 3 or ws.max_row < 4: + return # key-value / assumption sheets legitimately have no header row + for row in ws.iter_rows(min_row=1, max_row=min(ws.max_row, 10)): + cells = [c for c in row if c.value is not None] + if len(cells) < 2: + continue # a merged title row shows up as a single cell; skip past it + bad = [ + c.coordinate + for c in cells + if (isinstance(c.value, (int, float, dt.date, dt.datetime)) and not isinstance(c.value, bool)) + or (isinstance(c.value, str) and c.value.startswith("=")) + ] + if bad: + add( + "WARN", + f"工作表「{name}」第 {cells[0].row} 行是首个数据行，里面 {len(bad)}/{len(cells)} 格是数字或公式" + f"（{'、'.join(bad[:4])}）→ 这张表缺表头。在它上面插一行中文列名，并 freeze_panes 冻结。", + ) + return + + +def check_layout(ws, name: str, vs) -> None: + if ws.max_row <= 1 and ws.max_column <= 1 and ws["A1"].value is None: + add("WARN", f"工作表「{name}」是空的——要么填内容，要么删掉它") + return + + check_header(ws, name) + + # Column width vs content: the ### bug, and it hides Chinese first. + for col_idx in range(1, ws.max_column + 1): + letter = get_column_letter(col_idx) + dim = ws.column_dimensions.get(letter) + explicit = dim is not None and dim.width + width = float(dim.width) if explicit else float(ws.sheet_format.defaultColWidth or DEFAULT_COL_WIDTH) + need_num, need_text = 0, 0 + sample_num, sample_text = "", "" + for row_idx in range(1, min(ws.max_row, 500) + 1): + cell = ws.cell(row=row_idx, column=col_idx) + if cell.value is None: + continue + alignment = cell.alignment + if alignment is not None and alignment.wrap_text: + continue # wrapped text grows the row, not the column + cached = vs[cell.coordinate].value if vs is not None else None + text = rendered_text(cell, cached) + if not text: + continue + numeric = isinstance(cell.value, (int, float, dt.date, dt.datetime)) and not isinstance(cell.value, bool) + numeric = numeric or (isinstance(cell.value, str) and cell.value.startswith("=") and cached is not None) + # ### is a *formatted* number that will not fit. A General number + # never shows ### — Excel rounds the display instead — so it belongs + # in the softer bucket. + if numeric and (cell.number_format or "General") not in ("General", "@"): + if cjk_width(text) > need_num: + need_num, sample_num = cjk_width(text), text + elif cjk_width(text) > need_text: + need_text, sample_text = cjk_width(text), text + note = "" if explicit else "（没设列宽，用的是默认 8.43）" + if need_num and width < need_num: + add( + "WARN", + f"{name} 第 {letter} 列宽 {width:.0f}{note} < 数字渲染后的 {need_num}（如 {sample_num}，中文按双宽算）" + f"→ 打开就是 ###。调 ws.column_dimensions['{letter}'].width，或收尾时调 autofit_columns(ws)", + ) + elif need_text and width < need_text: + add( + "INFO", + f"{name} 第 {letter} 列宽 {width:.0f}{note} < 文字需要的 {need_text}（如 {sample_text}）" + f"→ 右边有内容时会被截断。调 autofit_columns(ws) 或给这列设 wrap_text", + ) + + # Merged ranges whose non-anchor cells carry values (openpyxl silently drops them). + for rng in ws.merged_cells.ranges: + min_col, min_row, max_col, max_row = range_boundaries(str(rng)) + for r in range(min_row, max_row + 1): + for c in range(min_col, max_col + 1): + if (r, c) == (min_row, min_col): + continue + cell = ws.cell(row=r, column=c) + if cell.value is not None: + add( + "WARN", + f"{name}!{rng} 合并区里非左上角单元格 {cell.coordinate} 仍有值，Excel 打开时会丢", + f"{name} 合并区非左上角有值", + ) + break + + if ws.max_row > 15 and not ws.freeze_panes: + add("INFO", f"工作表「{name}」超过 15 行但没冻结表头，滚动后看不到列名") + + +def check_number_formats(ws, name: str) -> None: + for row in ws.iter_rows(): + for cell in row: + if cell.value is None: + continue + fmt = cell.number_format or "General" + + # Percent format holding a value that is plainly not a fraction. + if "%" in fmt and isinstance(cell.value, (int, float)) and abs(cell.value) > 1.5: + add( + "ERROR", + f"{name}!{cell.coordinate} 百分比格式但存了 {cell.value} → 会显示成 " + f"{cell.value * 100:.0f}%。百分比要存小数（15% 存 0.15）", + f"{name} 百分比格式存了整数", + ) + + # Big bare numbers with no format read as a wall of digits. + if fmt == "General" and isinstance(cell.value, (int, float)) and abs(cell.value) >= 10000: + add( + "INFO", + f"{name}!{cell.coordinate} 金额 {cell.value} 没有数字格式，建议 `#,##0`", + f"{name} 第 {cell.column_letter} 列金额无数字格式", + ) + + # Years must not be thousands-separated. + if isinstance(cell.value, int) and 1900 <= cell.value <= 2100 and "#,##" in fmt: + add( + "WARN", + f"{name}!{cell.coordinate} 年份 {cell.value} 用了千分位格式，会显示成 {cell.value:,}", + f"{name} 年份用千分位", + ) + + +def run_checks(path: str) -> None: + wb = load_workbook(path, data_only=False) + try: + wb_values = load_workbook(path, data_only=True) + except Exception: + wb_values = None # a file this broken will surface elsewhere; keep the report alive + + sheet_names = set(wb.sheetnames) + total_formulas = 0 + uncached = 0 + + for name in wb.sheetnames: + ws = wb[name] + if not hasattr(ws, "iter_rows"): + continue # chartsheet + vs = wb_values[name] if wb_values is not None and name in wb_values.sheetnames else None + total_formulas += check_formulas(ws, name, sheet_names) + check_layout(ws, name, vs) + check_number_formats(ws, name) + if vs is not None: + for row in ws.iter_rows(): + for cell in row: + if not (isinstance(cell.value, str) and cell.value.startswith("=")): + continue + cached = vs[cell.coordinate].value + if cached is None: + uncached += 1 + elif isinstance(cached, str) and cached.strip() in ERROR_LITERALS: + # Recalculated to an error. Invisible in the formula pass + # (the cell still holds its formula text), so it must be + # read off the cached value or only recalc_check sees it. + add( + "ERROR", + f"{name}!{cell.coordinate} 重算结果是 {cached.strip()}：{fmt_value(cell.value)} " + f"→ 公式本身算不出来，改完再跑一次 recalc_check.py", + f"{name} 重算结果是错误值", + ) + + if uncached: + add( + "ERROR", + f"{uncached}/{total_formulas} 个公式没有缓存值——openpyxl 写出的公式不带结果，" + f"pandas 和多数预览器读到的是空。必须先跑 recalc_check.py 重算。", + ) + wb.close() + if wb_values is not None: + wb_values.close() + + print() + print("=" * 72) + print(f"体检 · 公式 {total_formulas} 个") + print("=" * 72) + + order = {"ERROR": 0, "WARN": 1, "INFO": 2} + counts = {"ERROR": 0, "WARN": 0, "INFO": 0} + for level, _, _ in findings: + counts[level] += 1 + + # One rule × one column can legitimately fire 500 times. Print a few examples + # per rule and count the rest, so the ERRORs stay visible above the noise. + sizes: dict[tuple[str, str], int] = {} + for level, _, group in findings: + sizes[(level, group)] = sizes.get((level, group), 0) + 1 + shown: dict[tuple[str, str], int] = {} + for level, message, group in sorted(findings, key=lambda f: order[f[0]]): + key = (level, group) + shown[key] = shown.get(key, 0) + 1 + if shown[key] <= MAX_PER_GROUP: + print(f"[{level}] {message}") + elif shown[key] == MAX_PER_GROUP + 1: + print(f"[{level}] …「{group}」同类还有 {sizes[key] - MAX_PER_GROUP} 处，按同样的办法一次改完") + if not findings: + print("（无发现）") + + print() + print(f"合计: {counts['ERROR']} ERROR / {counts['WARN']} WARN / {counts['INFO']} INFO") + # Only ERROR gates delivery, and the pass line must be printable whenever + # ERROR is zero. An extra "conditional pass" tier for WARN reads as helpful + # but is a trap: SKILL.md tells the model to iterate until `结论: 通过` + # appears, while several WARN rules here are heuristics that a legitimate + # workbook can never clear (a genuinely header-less matrix, the ±2 error of + # the rendered-width estimate). The model would then loop until it runs out + # of turns. Same wording as bisheng-pptx / bisheng-docx on purpose. + if counts["ERROR"]: + print("结论: 不通过 —— ERROR 必须全部修完再交付。") + else: + print("结论: 通过 —— 无必须修复项。WARN 逐条复核后即可交付。") + + +def main() -> None: + parser = argparse.ArgumentParser(description="读出并体检一个 .xlsx") + parser.add_argument("path") + parser.add_argument("--content-only", action="store_true", help="只读内容，不体检") + parser.add_argument("--check-only", action="store_true", help="只体检，不打印内容") + parser.add_argument("--max-rows", type=int, default=40, help="每个表最多打印多少行（默认 40）") + args = parser.parse_args() + + if not os.path.exists(args.path): + print(f"[FATAL] 文件不存在: {args.path}") + print("提示：代码执行器 cwd 就是工作区根，用相对路径 `output/x.xlsx`，不要写 `/output/x.xlsx`。") + return + + try: + if not args.check_only: + dump_content(args.path, args.max_rows) + if not args.content_only: + run_checks(args.path) + except Exception as exc: + # Report and still exit 0 — a traceback on stdout is useful; a non-zero + # exit would make the executor discard this whole report. + print("[FATAL] 体检过程本身出错：") + if "not a zip file" in str(exc).lower(): + print("这个文件不是真的 .xlsx（.xlsx 本质是 zip）。旧版 .xls 或改了扩展名的 csv 会这样。") + print("改法：.xls/.csv 用 pandas.read_excel / read_csv 读进来，再用 openpyxl 写一份新的 .xlsx。") + traceback.print_exc(file=sys.stdout) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/recalc_check.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/recalc_check.py new file mode 100644 index 0000000000..2da13491bd --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/recalc_check.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Recalculate an openpyxl-written .xlsx with LibreOffice, then report errors. + + python skills/bisheng-xlsx/scripts/recalc_check.py output/x.xlsx [--timeout 60] + +Why this is mandatory whenever the workbook contains formulas: openpyxl writes +formulas as strings with **no cached value**. Until something evaluates them, +every formula cell reads back as ``None`` to pandas, to +``load_workbook(data_only=True)``, and to most previewers — the file looks empty +where the numbers should be. + +LibreOffice evaluates them via a StarBasic macro installed into a throwaway +profile, rewrites the file in place, and this script then re-reads it and names +every cell that came back as an Excel error literal. + +BiSheng specifics: + +* **Always exits 0.** The executor keeps stdout only on a zero exit; exiting + non-zero on "errors found" would delete the very report that names them. +* **Private profile per run**, so four concurrent conversions do not collide + (a fixed profile path is the classic multi-task corruption bug). +* Degrades with an explicit message when LibreOffice has no Calc component — + some hand-built hosts install only ``libreoffice-writer``, and the resulting + failure otherwise masquerades as a timeout. +* **Never prints ``结论: 通过``.** SKILL.md tells the model to iterate until that + exact string appears, and it is owned by ``inspect_workbook.py`` — the delivery + gate. Recalculation is only one step before it, so a passing recalc says + ``重算完成`` instead. Emitting the reserved string here would let the model stop + one step early, shipping a workbook that was never health-checked. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import traceback +import uuid +from pathlib import Path + +try: + from openpyxl import load_workbook +except ImportError: # pragma: no cover + print("[FATAL] openpyxl 不可用，无法重算。") + sys.exit(0) + +MACRO = """ + + + Sub RecalculateAndSave() + ThisComponent.calculateAll() + ThisComponent.store() + ThisComponent.close(True) + End Sub +""" + +ERROR_LITERALS = ["#VALUE!", "#DIV/0!", "#REF!", "#NAME?", "#NULL!", "#NUM!", "#N/A"] +MAX_LOCATIONS = 50 + +# '[1]Sheet'!$B$2 — an index into the workbook's external-reference list, i.e. a +# separate file on disk. openpyxl drops the cached value on save; LibreOffice +# then cannot resolve the link and bakes in #NAME?, destroying the data. +EXTERNAL_REF_RE = __import__("re").compile(r"""(? str | None: + return shutil.which("soffice") or shutil.which("libreoffice") + + +def calc_binary_found(soffice: str) -> bool: + """Whether an ``scalc`` launcher is visible next to this soffice. + + ADVISORY ONLY — never a gate. Calc ships as a separate package on some + hosts, so a missing ``scalc`` is worth mentioning *after* a failure, but a + present Calc is not guaranteed to expose that file: macOS bundles ship a + single ``Contents/MacOS/soffice`` with no per-app launchers at all, and + refusing to convert there was a 100% false negative. Try first, diagnose + after. + """ + root = Path(soffice).resolve().parent.parent + for candidate in (root / "lib" / "libreoffice" / "program", root / "program", Path("/usr/lib/libreoffice/program")): + if (candidate / "scalc").exists() or (candidate / "scalc.bin").exists(): + return True + return False + + +def external_links(path: str) -> list[str]: + hits = [] + wb = load_workbook(path, data_only=False) + try: + for name in wb.sheetnames: + ws = wb[name] + if not hasattr(ws, "iter_rows"): + continue + for row in ws.iter_rows(): + for cell in row: + if isinstance(cell.value, str) and EXTERNAL_REF_RE.search(cell.value): + hits.append(f"{name}!{cell.coordinate}") + if len(hits) >= 10: + return hits + finally: + wb.close() + return hits + + +def install_macro(profile: Path, soffice: str, timeout: int) -> str | None: + """Boot LibreOffice once so it materialises the profile, then drop the macro in.""" + url = profile.as_uri() + try: + subprocess.run( + [soffice, "--headless", "--terminate_after_init", f"-env:UserInstallation={url}"], + capture_output=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return None + macro_dir = profile / "user" / "basic" / "Standard" + if not macro_dir.exists(): + return None + (macro_dir / "Module1.xba").write_text(MACRO) + return url + + +def scan(path: str) -> dict: + wb = load_workbook(path, data_only=True) + details: dict[str, list[str]] = {e: [] for e in ERROR_LITERALS} + total = 0 + for name in wb.sheetnames: + ws = wb[name] + if not hasattr(ws, "iter_rows"): + continue + for row in ws.iter_rows(): + for cell in row: + if isinstance(cell.value, str): + for err in ERROR_LITERALS: + if err in cell.value: + details[err].append(f"{name}!{cell.coordinate}") + total += 1 + break + wb.close() + + wb_f = load_workbook(path, data_only=False) + formulas = 0 + uncached = 0 + wb_v = load_workbook(path, data_only=True) + for name in wb_f.sheetnames: + ws = wb_f[name] + if not hasattr(ws, "iter_rows"): + continue + vs = wb_v[name] + for row in ws.iter_rows(): + for cell in row: + if isinstance(cell.value, str) and cell.value.startswith("="): + formulas += 1 + if vs[cell.coordinate].value is None: + uncached += 1 + wb_f.close() + wb_v.close() + + summary = {} + for err, locations in details.items(): + if locations: + entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]} + if len(locations) > MAX_LOCATIONS: + entry["locations_truncated"] = len(locations) - MAX_LOCATIONS + summary[err] = entry + + return { + "status": "success" if total == 0 else "errors_found", + "total_formulas": formulas, + "total_errors": total, + "uncached_formulas": uncached, + "error_summary": summary, + } + + +def recalc(path: str, timeout: int, force: bool) -> dict: + soffice = find_soffice() + if not soffice: + return {"error": "环境里没有 soffice（LibreOffice）——无法重算公式。"} + calc_hint = ( + "" + if calc_binary_found(soffice) + else "另：这套 LibreOffice 旁边找不到 scalc，若反复失败就是没装 Calc 组件（手工部署常只装 libreoffice-writer），让运维补装 libreoffice-calc。" + ) + + links = external_links(path) + if links and not force: + return { + "error": ( + f"这个工作簿引用了外部文件（如 {', '.join(links[:3])}）。openpyxl 保存时已丢掉缓存值，" + f"重算会让 LibreOffice 解析不到外链、写入 #NAME? 并删掉链接，数据就没了。" + f"先把这些单元格的值从原件里取出来固化，再重算；确认可以承受损失时加 --force。" + ) + } + + abs_path = os.path.abspath(path) + before = os.stat(abs_path) + before_stamp = (before.st_mtime_ns, before.st_size) + + with tempfile.TemporaryDirectory(prefix=f"lo_recalc_{uuid.uuid4().hex[:8]}_") as tmp: + profile = Path(tmp) / "profile" + started = time.monotonic() + url = install_macro(profile, soffice, timeout) + if url is None: + return {"error": f"LibreOffice 没能在 {timeout}s 内建出可用的 profile，公式未重算。"} + + remaining = max(10, int(timeout - (time.monotonic() - started))) + cmd = [ + soffice, + "--headless", + "--norestore", + "--invisible", + "-env:SingleAppInstance=false", + f"-env:UserInstallation={url}", + "vnd.sun.star.script:Standard.Module1.RecalculateAndSave?language=Basic&location=application", + abs_path, + ] + try: + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=remaining + 15) + except subprocess.TimeoutExpired: + return {"error": f"LibreOffice 重算超时（{remaining}s）。加大 --timeout 重试。{calc_hint}"} + + # The file is the only evidence that counts. soffice exits 0 after doing + # nothing at all, and it exits non-zero on shutdown noise (macOS prints + # "Task policy set failed") after a perfectly good rewrite — so read the + # stat first and treat the return code as commentary. + after = os.stat(abs_path) + rewritten = (after.st_mtime_ns, after.st_size) != before_stamp + + if not rewritten: + detail = ((proc.stderr or "") + (proc.stdout or "")).strip() + hint = "" + if proc.returncode == 137: + hint = "（退出码 137 = 被 OOM killer 杀掉，多半是内存配额太小，不是 LibreOffice 坏了）" + elif "could not be loaded" in detail: + hint = "（报了 could not be loaded = 这套 LibreOffice 打不开 .xlsx，基本就是缺 Calc 组件）" + return { + "error": ( + f"LibreOffice 退出码 {proc.returncode} 但文件没被重写，等于什么都没算{hint}。" + f"soffice 输出：{detail[:400] or '（空）'}。" + f"重试一次；仍失败就把文件另存一个新名字再试。{calc_hint}" + ) + } + + result = scan(path) + if proc.returncode != 0: + result["soffice_returncode"] = proc.returncode + return result + + +def report(result: dict) -> None: + print("=" * 72) + print("公式重算") + print("=" * 72) + + if "error" in result: + print(f"[未重算] {result['error']}") + print() + print("重算未通过 —— 公式没有缓存值，交付出去用户和 pandas 读到的都是空。") + print("降级方案：如果这台机器确实没有 Calc，就不要用公式，直接在 Python 里算好写入数值，") + print("并在表里用一列文字说明计算口径（否则用户改了输入也不会重算）。") + print() + print("JSON: " + json.dumps(result, ensure_ascii=False)) + return + + print(f"公式总数 : {result['total_formulas']}") + print(f"错误单元格: {result['total_errors']}") + if result.get("uncached_formulas"): + print(f"仍无缓存值: {result['uncached_formulas']}（异常，重算应当把它们都填上）") + + for err, entry in result.get("error_summary", {}).items(): + locations = ", ".join(entry["locations"]) + extra = f"（另有 {entry['locations_truncated']} 处未列出）" if "locations_truncated" in entry else "" + print(f" {err} × {entry['count']}: {locations}{extra}") + + print() + if result["total_errors"]: + print("重算未通过 —— 上面每个错误单元格都要改完再重算。") + else: + print("重算完成 —— 公式都能算出结果。") + print("注意：能算 ≠ 算对。口径错、区间差一行照样是干净的错数字 ——") + print("先抽查 2–3 个关键公式的结果符不符合预期，再铺开整张表。") + print() + print("JSON: " + json.dumps(result, ensure_ascii=False)) + + +def main() -> None: + parser = argparse.ArgumentParser(description="用 LibreOffice 重算 xlsx 公式并检查错误") + parser.add_argument("path") + parser.add_argument("--timeout", type=int, default=60) + parser.add_argument("--force", action="store_true", help="即使存在外部链接也强行重算（接受数据丢失）") + args = parser.parse_args() + + if not os.path.exists(args.path): + print(f"[FATAL] 文件不存在: {args.path}") + print("提示：用相对路径 `output/x.xlsx`，不要写 `/output/x.xlsx`。") + return + + try: + report(recalc(args.path, args.timeout, args.force)) + except Exception as exc: + print("[FATAL] 重算过程本身出错：") + if "not a zip file" in str(exc).lower(): + print("这个文件不是真的 .xlsx（.xlsx 本质是 zip）。旧版 .xls 或改了扩展名的 csv 会这样。") + print("改法：.xls/.csv 用 pandas 读进来，再用 openpyxl 写一份新的 .xlsx，然后重算这一份。") + traceback.print_exc(file=sys.stdout) + + +if __name__ == "__main__": + main() + sys.exit(0) diff --git a/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/xlsx_helpers.py b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/xlsx_helpers.py new file mode 100644 index 0000000000..d2837bb4fe --- /dev/null +++ b/src/backend/bisheng/linsight/builtin_skills/bisheng-xlsx/scripts/xlsx_helpers.py @@ -0,0 +1,196 @@ +"""openpyxl helpers for building .xlsx inside the BiSheng code interpreter. + +Use from a build script: + + import sys + sys.path.insert(0, "skills/bisheng-xlsx/scripts") + from xlsx_helpers import autofit_columns, write_table, mark_input, FMT + +Everything here is plain openpyxl — no dependency the backend image does not +already ship. The value is in the traps it closes: CJK column width, percentages +stored as whole numbers, sheet names needing quotes, and the audit colour +convention, each of which is a silent wrong-output bug rather than a crash. +""" + +from openpyxl.comments import Comment +from openpyxl.styles import Alignment, Border, Font, PatternFill, Side +from openpyxl.utils import get_column_letter + +# --- Audit colour convention ------------------------------------------------- +# Blue = a number a human typed. Black = a formula. Green = another sheet. +# Red = another file. Reviewers read the colour before they read the number, so +# getting this wrong makes a model look hand-checked when it was not. +BLUE = "0000FF" +BLACK = "000000" +GREEN = "008000" +RED = "FF0000" +YELLOW_FILL = "FFFF00" + +HEADER_FILL = "1F4E79" +HEADER_FONT_COLOR = "FFFFFF" +BAND_FILL = "F2F6FA" + +# Default face. Keep it to fonts a Chinese user's Excel definitely has — +# rendering happens on their machine, not ours. +FONT_NAME = "微软雅黑" +FONT_NAME_LATIN = "Arial" + +FMT = { + "money": "#,##0;(#,##0);-", + "money_cny": "¥#,##0;(¥#,##0);-", + "money_mm": "#,##0.0;(#,##0.0);-", + "int": "#,##0;(#,##0);-", + "pct": "0.0%", + "pct0": "0%", + "mult": "0.0x", + "date": "yyyy-mm-dd", + "text": "@", +} + +THIN = Side(style="thin", color="BFBFBF") +BORDER_ALL = Border(left=THIN, right=THIN, top=THIN, bottom=THIN) + + +def cjk_width(text) -> int: + """Display width in half-widths: CJK/full-width glyphs count as 2. + + ``len()`` under-measures Chinese by half, which is exactly why auto-fitted + columns come out too narrow and numbers render as ``###``. + """ + if text is None: + return 0 + width = 0 + for ch in str(text): + # CJK ideographs, kana, full-width forms, and CJK punctuation. + if "一" <= ch <= "鿿" or "　" <= ch <= "〿" or "぀" <= ch <= "ヿ" or "＀" <= ch <= "￯": + width += 2 + else: + width += 1 + return width + + +def autofit_columns(ws, min_width: int = 8, max_width: int = 60, padding: int = 3) -> None: + """Set column widths from the widest cell, counting CJK as double width.""" + widest: dict[int, int] = {} + for row in ws.iter_rows(): + for cell in row: + if cell.value is None: + continue + # A formula's own text says nothing about how wide its result is; + # reserve a sane default instead of measuring "=SUM(B2:B99)". + text = "0,000,000" if isinstance(cell.value, str) and cell.value.startswith("=") else cell.value + widest[cell.column] = max(widest.get(cell.column, 0), cjk_width(text)) + for col, width in widest.items(): + ws.column_dimensions[get_column_letter(col)].width = max(min_width, min(max_width, width + padding)) + + +def style_header(ws, row: int = 1, first_col: int = 1, last_col: int | None = None) -> None: + """Dark fill + white bold text + centred, the convention reviewers expect.""" + last_col = last_col or ws.max_column + for col in range(first_col, last_col + 1): + cell = ws.cell(row=row, column=col) + cell.font = Font(name=FONT_NAME, bold=True, color=HEADER_FONT_COLOR, size=11) + cell.fill = PatternFill("solid", fgColor=HEADER_FILL) + cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) + cell.border = BORDER_ALL + ws.row_dimensions[row].height = 22 + + +def write_table(ws, headers, rows, start_row: int = 1, start_col: int = 1, number_format=None, banded=True): + """Write a header row + data rows and return the first/last data row numbers. + + ``number_format`` maps a 0-based column offset to a key of :data:`FMT` (or a + raw format string), e.g. ``{1: "money", 2: "pct"}``. + """ + number_format = number_format or {} + for offset, title in enumerate(headers): + ws.cell(row=start_row, column=start_col + offset, value=title) + style_header(ws, row=start_row, first_col=start_col, last_col=start_col + len(headers) - 1) + + for r, record in enumerate(rows, start=start_row + 1): + for offset, value in enumerate(record): + cell = ws.cell(row=r, column=start_col + offset, value=value) + cell.font = Font(name=FONT_NAME, size=10) + cell.border = BORDER_ALL + key = number_format.get(offset) + if key: + cell.number_format = FMT.get(key, key) + if banded and (r - start_row) % 2 == 0: + cell.fill = PatternFill("solid", fgColor=BAND_FILL) + first_data_row = start_row + 1 + return first_data_row, start_row + len(rows) + + +def set_pct(cell, fraction, fmt: str = "pct") -> None: + """Write a percentage. Pass 0.15 for 15%. + + Storing 15 with a percent format renders 1500.0%. This wrapper refuses the + ambiguous range loudly instead of shipping a wrong-by-100x workbook. + """ + if fraction is not None and abs(float(fraction)) > 1.5: + raise ValueError( + f"set_pct got {fraction!r}: percentages are stored as fractions " + f"(0.15 == 15%). Pass {float(fraction) / 100!r} instead, or use " + f"cell.number_format directly if this really is {fraction}x." + ) + cell.value = fraction + cell.number_format = FMT.get(fmt, fmt) + + +def mark_input(cell, note: str | None = None) -> None: + """A number a human typed: blue. Optionally attach the source as a comment.""" + cell.font = Font(name=FONT_NAME, size=10, color=BLUE) + if note: + add_note(cell, note) + + +def mark_formula(cell) -> None: + cell.font = Font(name=FONT_NAME, size=10, color=BLACK) + + +def mark_cross_sheet(cell) -> None: + cell.font = Font(name=FONT_NAME, size=10, color=GREEN) + + +def mark_assumption(cell, note: str | None = None) -> None: + """A key assumption / a cell the user is meant to fill in: yellow fill.""" + cell.fill = PatternFill("solid", fgColor=YELLOW_FILL) + cell.font = Font(name=FONT_NAME, size=10, color=BLUE) + if note: + add_note(cell, note) + + +def add_note(cell, text: str, author: str = "BiSheng") -> None: + """Attach a cell comment. This is where an assumption's source belongs.""" + comment = Comment(text, author) + comment.width = 260 + comment.height = 110 + cell.comment = comment + + +def sheet_ref(sheet_name: str, coord: str) -> str: + """Build a cross-sheet reference, quoting the name when it needs quoting. + + ``='Assumptions Inputs'!$B$5`` evaluates; the unquoted form yields #VALUE!. + """ + needs_quotes = any(ch in sheet_name for ch in " -()&'") or not sheet_name.isascii() + name = "'" + sheet_name.replace("'", "''") + "'" if needs_quotes else sheet_name + return f"={name}!{coord}" + + +def freeze_and_filter(ws, header_row: int = 1, add_filter: bool = True) -> None: + """Freeze everything above the first data row and turn on autofilter.""" + ws.freeze_panes = ws.cell(row=header_row + 1, column=1) + if add_filter and ws.max_row > header_row: + ws.auto_filter.ref = f"A{header_row}:{get_column_letter(ws.max_column)}{ws.max_row}" + + +def add_legend(ws, cell_coord: str, lines) -> None: + """Drop a short legend block. A workbook someone must fill in needs one.""" + row = ws[cell_coord].row + col = ws[cell_coord].column + ws.cell(row=row, column=col, value="填写说明").font = Font(name=FONT_NAME, bold=True, size=10) + for offset, line in enumerate(lines, start=1): + cell = ws.cell(row=row + offset, column=col, value=line) + cell.font = Font(name=FONT_NAME, size=9, color="595959") + cell.alignment = Alignment(vertical="top", wrap_text=True) diff --git a/src/backend/bisheng/linsight/domain/models/linsight_skill.py b/src/backend/bisheng/linsight/domain/models/linsight_skill.py index 3e5f6cf8e7..5c43e69f48 100644 --- a/src/backend/bisheng/linsight/domain/models/linsight_skill.py +++ b/src/backend/bisheng/linsight/domain/models/linsight_skill.py @@ -16,14 +16,20 @@ # Skill source markers (C3/C7 contract). SKILL_SOURCE_MANUAL = "manual" SKILL_SOURCE_SOP_MIGRATED = "sop_migrated" +# Shipped with the kernel and seeded into every tenant at startup +# (services/builtin_skill_seeder.py). Editing one through the API flips it to +# ``manual``, which permanently opts that tenant's copy out of re-seeding. +SKILL_SOURCE_BUILTIN = "builtin" class LinsightSkillBase(SQLModelSerializable): """Tenant-scoped custom skill metadata (F035). The skill body lives on disk under ``SKILLS_ROOT/data/skills/{tenant_id}//SKILL.md`` - (see design §7.1); this table only owns the metadata. Built-in skills are - never persisted here and never exposed through the ``/skill`` API. + (see design §7.1); this table only owns the metadata. Kernel built-in skills + are seeded into this same table (``source='builtin'``) so they share one + runtime path with uploaded ones — the picker, the enable/disable toggle and + ``materialize_session_skills`` need no special case for them. """ tenant_id: int = Field( @@ -52,7 +58,7 @@ class LinsightSkillBase(SQLModelSerializable): ) source: str = Field( default=SKILL_SOURCE_MANUAL, - description="manual | sop_migrated", + description="manual | sop_migrated | builtin", sa_column=Column( String(32), nullable=False, server_default=text(f"'{SKILL_SOURCE_MANUAL}'"), comment="Skill origin" ), diff --git a/src/backend/bisheng/linsight/domain/services/agent_factory.py b/src/backend/bisheng/linsight/domain/services/agent_factory.py index 2469151745..804571545d 100644 --- a/src/backend/bisheng/linsight/domain/services/agent_factory.py +++ b/src/backend/bisheng/linsight/domain/services/agent_factory.py @@ -354,9 +354,9 @@ def _build_linsight_system_prompt( "写会被丢弃、进不了交付。\n" "- 反向同理：**不要**把代码里看到的宿主机绝对路径（形如 /root/.cache/…/<8位任务号>/output/a.png）" "传给 read_file / edit_file，去掉前缀只传 output/a.png。\n" - "- 执行器直接写本地工作目录，它生成的文件**不会**出现在 ls / glob 的结果里" - "（那两个工具读的是对象存储视图）。只要执行返回 exitcode 0 且日志显示写成功，" - "就视为交付物已产出，继续下一步；不要反复 ls / glob 找它，也不要因为“找不到”而重新生成。\n\n" + "- 执行器写完会把本轮产出同步到工作区，随后 ls / glob 一般就能看到它们。" + "但**判定成功的依据是执行结果本身**：只要 exitcode 0 且日志显示写成功，就视为交付物已产出，" + "继续下一步；不要反复 ls / glob 找它，更不要因为一时“找不到”而重新生成一遍。\n\n" ) return ( diff --git a/src/backend/bisheng/linsight/domain/services/builtin_skill_seeder.py b/src/backend/bisheng/linsight/domain/services/builtin_skill_seeder.py new file mode 100644 index 0000000000..f5320b4b8a --- /dev/null +++ b/src/backend/bisheng/linsight/domain/services/builtin_skill_seeder.py @@ -0,0 +1,209 @@ +"""Install the kernel's built-in skill bundles into every tenant at startup. + +A shipped skill has to reach the same place a user-uploaded one does — disk under +``SKILLS_ROOT/data/skills/{tenant_id}//`` plus a ``linsight_skill`` row — +because that is the only path the picker, the governance toggle and +``materialize_session_skills`` know about. Seeding there means an out-of-the-box +deployment shows the official skills with zero operator action, and every existing +capability (enable/disable, detail view, per-tenant isolation) works unchanged. + +Why here and not in a migration: project law says an Alembic revision does DDL +only — any data seeding/backfill is a separate operational step. Doing it in the +API lifespan keeps `docker compose up` a single command while staying out of the +migration chain. + +Idempotency is content-based: the bundle on disk is compared byte-for-byte with +the one shipped in the image, so an upgraded image updates the skill on the next +restart and an unchanged one costs a few file reads. A tenant that *edited* a +built-in skill has its row flipped to ``manual`` by the update endpoints, and +this seeder then leaves it alone forever — silently reverting a customer's edits +on upgrade would be far worse than letting their copy drift. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path + +from loguru import logger + +from bisheng.core.context.tenant import DEFAULT_TENANT_ID, current_tenant_id, set_current_tenant_id +from bisheng.database.models.tenant import TenantDao +from bisheng.linsight.domain.models.linsight_skill import ( + SKILL_SOURCE_BUILTIN, + LinsightSkill, + LinsightSkillDao, +) +from bisheng.linsight.domain.services.skill_store import ( + DISPLAY_NAME_META_KEY, + SKILL_MD, + SkillStore, + parse_skill_md, + validate_skill_name, +) + +# ``bisheng/linsight/builtin_skills//`` — inside the package so every +# deployment shape (docker COPY, rsync, pip install) carries it automatically. +BUILTIN_SKILLS_DIR = Path(__file__).resolve().parents[2] / "builtin_skills" + +_SKIP_PARTS = {"__pycache__", ".git"} +_SKIP_NAMES = {".DS_Store"} + + +def _read_bundle(bundle_dir: Path) -> tuple[dict, dict[str, bytes]] | None: + """``(frontmatter, {relative_path: bytes})`` for one bundle dir, or None if unusable.""" + files: dict[str, bytes] = {} + for path in sorted(bundle_dir.rglob("*")): + if path.is_dir() or path.name in _SKIP_NAMES or _SKIP_PARTS & set(path.parts): + continue + files[path.relative_to(bundle_dir).as_posix()] = path.read_bytes() + + if SKILL_MD not in files: + logger.warning("built-in skill {} has no {}; skipped", bundle_dir.name, SKILL_MD) + return None + try: + meta, _ = parse_skill_md(files[SKILL_MD].decode("utf-8")) + except (ValueError, UnicodeDecodeError) as exc: + logger.warning("built-in skill {} has an unreadable {}: {}", bundle_dir.name, SKILL_MD, exc) + return None + + name = str(meta.get("name") or "").strip() + # The bundle directory name IS the skill id at runtime (deepagents resolves + # skills by path), so a mismatch would install something the model cannot read. + if name != bundle_dir.name or validate_skill_name(name) is not None: + logger.warning( + "built-in skill dir {!r} does not match a valid frontmatter name {!r}; skipped", + bundle_dir.name, + name, + ) + return None + if not str(meta.get("description") or "").strip(): + logger.warning("built-in skill {} has no description; skipped", name) + return None + return meta, files + + +def discover_builtin_bundles() -> dict[str, tuple[dict, dict[str, bytes]]]: + """All shipped bundles, keyed by skill name.""" + if not BUILTIN_SKILLS_DIR.is_dir(): + return {} + found: dict[str, tuple[dict, dict[str, bytes]]] = {} + for child in sorted(BUILTIN_SKILLS_DIR.iterdir()): + if not child.is_dir() or child.name in _SKIP_PARTS: + continue + parsed = _read_bundle(child) + if parsed: + found[child.name] = parsed + return found + + +def _installed_matches(store: SkillStore, tenant_id: int, name: str, files: dict[str, bytes]) -> bool: + """True when the on-disk bundle is byte-identical to the shipped one.""" + try: + installed = { + entry["path"]: store.read_bytes(tenant_id, name, entry["path"]) + for entry in store.list_files(tenant_id, name) + } + except Exception: # missing dir, unreadable file — treat as "needs rewrite" + return False + return installed == files + + +def _display_name_of(meta: dict, name: str) -> str: + metadata = meta.get("metadata") + if isinstance(metadata, dict): + value = str(metadata.get(DISPLAY_NAME_META_KEY) or "").strip() + if value: + return value + return name + + +async def _seed_one(store: SkillStore, tenant_id: int, name: str, meta: dict, files: dict[str, bytes]) -> str: + """Install/refresh one bundle for one tenant. Returns the outcome label.""" + existing = await LinsightSkillDao.get_by_name(name) + + if existing and existing.source != SKILL_SOURCE_BUILTIN: + # The tenant forked it (any edit through the API flips source to manual). + return "forked" + if existing and _installed_matches(store, tenant_id, name, files): + return "unchanged" + + size = store.write_bundle(tenant_id, name, files) + display_name = _display_name_of(meta, name) + description = str(meta["description"]).strip() + + if existing: + existing.display_name = display_name + existing.description = description + existing.size = size + existing.object_path = store.object_path(tenant_id, name) + await LinsightSkillDao.update(existing) + return "updated" + + try: + await LinsightSkillDao.create( + LinsightSkill( + tenant_id=tenant_id, + name=name, + display_name=display_name, + description=description, + enabled=True, + source=SKILL_SOURCE_BUILTIN, + object_path=store.object_path(tenant_id, name), + size=size, + ) + ) + return "created" + except Exception: + # Several API replicas boot at once; uq_linsight_skill_tenant_name makes + # the loser's INSERT fail, and the winner already wrote the same bytes. + logger.debug("built-in skill {!r} insert lost a race for tenant {}", name, tenant_id) + return "raced" + + +async def seed_builtin_skills(tenant_ids: Iterable[int] | None = None, *, store: SkillStore | None = None) -> dict: + """Seed every shipped bundle into the given tenants (default: all active ones). + + Best-effort by contract: this runs inside application startup, so any failure + is logged and swallowed — a broken bundle or an unreachable DB must never stop + the service from coming up. + """ + bundles = discover_builtin_bundles() + if not bundles: + return {} + + if tenant_ids is None: + try: + tenant_ids = await TenantDao.aget_active_ids() + except Exception: + logger.exception("built-in skill seeding could not list tenants; falling back to the default tenant") + tenant_ids = set() + # Single-tenant deployments have no rows in the tenant table at all. + tenant_ids = tenant_ids or {DEFAULT_TENANT_ID} + + store = store or SkillStore() + stats: dict[str, int] = {} + for tenant_id in sorted(tenant_ids): + token = set_current_tenant_id(tenant_id) + try: + for name, (meta, files) in bundles.items(): + try: + outcome = await _seed_one(store, tenant_id, name, meta, files) + except Exception: + logger.exception("built-in skill {!r} failed to seed for tenant {}", name, tenant_id) + outcome = "failed" + stats[outcome] = stats.get(outcome, 0) + 1 + finally: + current_tenant_id.reset(token) + + changed = stats.get("created", 0) + stats.get("updated", 0) + if changed: + logger.info( + "built-in skills seeded: {} bundle(s) x {} tenant(s) -> {}", + len(bundles), + len(list(tenant_ids)), + stats, + ) + else: + logger.debug("built-in skills already up to date: {}", stats) + return stats diff --git a/src/backend/bisheng/linsight/domain/services/skill_service.py b/src/backend/bisheng/linsight/domain/services/skill_service.py index b545a3d168..c949c14285 100644 --- a/src/backend/bisheng/linsight/domain/services/skill_service.py +++ b/src/backend/bisheng/linsight/domain/services/skill_service.py @@ -24,6 +24,7 @@ ) from bisheng.common.schemas.api import PageData from bisheng.linsight.domain.models.linsight_skill import ( + SKILL_SOURCE_BUILTIN, SKILL_SOURCE_MANUAL, LinsightSkill, LinsightSkillDao, @@ -199,6 +200,18 @@ async def create_from_github(self, tenant_id: int, user_id: int, url: str) -> Sk detail.normalized_from = meta.normalized_from return detail + @staticmethod + def _mark_forked(skill) -> None: + """An edited built-in skill becomes this tenant's own copy. + + The startup seeder refreshes ``source='builtin'`` rows from the image on + every boot. Leaving the marker on an edited skill would silently revert + the customer's changes at the next upgrade, so the first edit opts that + copy out of re-seeding for good. + """ + if skill.source == SKILL_SOURCE_BUILTIN: + skill.source = SKILL_SOURCE_MANUAL + async def update_from_form(self, tenant_id: int, name: str, form: SkillCreateForm) -> SkillDetail: skill = await self._get_or_404(name) if form.name != name: @@ -213,6 +226,7 @@ async def update_from_form(self, tenant_id: int, name: str, form: SkillCreateFor files[SKILL_MD] = new_md size = self.store.write_bundle(tenant_id, name, files) skill.display_name, skill.description, skill.size = form.display_name, form.description, size + self._mark_forked(skill) await LinsightSkillDao.update(skill) return await self.get_detail(tenant_id, name) @@ -225,6 +239,7 @@ async def update_from_upload(self, tenant_id: int, name: str, filename: str, dat await self._check_duplicate(name, meta.display_name, exclude_id=skill.id) size = self.store.write_bundle(tenant_id, name, files) skill.display_name, skill.description, skill.size = meta.display_name, meta.description, size + self._mark_forked(skill) await LinsightSkillDao.update(skill) detail = await self.get_detail(tenant_id, name) detail.normalized_from = meta.normalized_from diff --git a/src/backend/bisheng/linsight/domain/services/workbench_impl.py b/src/backend/bisheng/linsight/domain/services/workbench_impl.py index 24257371e4..9ea0607de2 100644 --- a/src/backend/bisheng/linsight/domain/services/workbench_impl.py +++ b/src/backend/bisheng/linsight/domain/services/workbench_impl.py @@ -1472,7 +1472,9 @@ async def _cache_parse_result(cls, file_id: str, parse_result: dict) -> None: await redis_client.aset(key=key, value=parse_result, expiration=60 * 60 * cls.CACHE_EXPIRATION_HOURS) @classmethod - async def _init_bisheng_code_tool(cls, selected_tool_ids: list[int], file_dir: str, user_id: int) -> list[BaseTool]: + async def _init_bisheng_code_tool( + cls, selected_tool_ids: list[int], file_dir: str, user_id: int, session_version_id: str | None = None + ) -> list[BaseTool]: """Initialize the code interpreter separately (it needs the workspace dir bound). ``selected_tool_ids`` is the user's per-turn tool selection. The id is @@ -1491,6 +1493,14 @@ async def _init_bisheng_code_tool(cls, selected_tool_ids: list[int], file_dir: s if "local" not in code_config["config"]: code_config["config"]["local"] = {} code_config["config"]["local"]["local_sync_path"] = file_dir + if session_version_id: + from bisheng.linsight.domain.services.workspace_backend import WORKSPACE_PREFIX + + # Lets the executor mirror what it writes into ``workspace//``. + # Without it the local working dir is the ONLY copy, so the file tools + # cannot see a code-generated deliverable and the next turn's + # seed-from-previous finds an empty ``output/``. + code_config["config"]["local"]["workspace_prefix"] = f"{WORKSPACE_PREFIX}/{session_version_id}" if "e2b" not in code_config["config"]: code_config["config"]["e2b"] = {} code_config["config"]["e2b"]["local_sync_path"] = file_dir @@ -1574,7 +1584,10 @@ async def init_linsight_config_tools( if need_upload and file_dir: try: bisheng_code_tool = await cls._init_bisheng_code_tool( - tool_ids, file_dir, user_id=session_version.user_id + tool_ids, + file_dir, + user_id=session_version.user_id, + session_version_id=session_version.id, ) tools.extend(bisheng_code_tool) except Exception: diff --git a/src/backend/bisheng/linsight/worker.py b/src/backend/bisheng/linsight/worker.py index d9b85dbbec..c0c54a2e09 100644 --- a/src/backend/bisheng/linsight/worker.py +++ b/src/backend/bisheng/linsight/worker.py @@ -138,16 +138,22 @@ async def start_heartbeat(self): await asyncio.sleep(self.interval) async def register_task_ownership(self, session_version_id): - """Register task ownership to this node""" + """Register task ownership to this node. + + The key is never deleted explicitly — it expires via its TTL, and a + re-queued task (resume / follow-up) overwrites it with the node that + picked it up. Do NOT add a release-on-park counterpart: park-and-release + frees the concurrency semaphore, not the ownership key. Deleting the key + while the session row is still IN_PROGRESS makes the worker-startup sweep + (``check_and_terminate_incomplete_tasks``) hit its ``not owner_node_id`` + branch and fail the task with "Worker node crash detected". Parked tasks + stay out of that sweep via the WAITING_FOR_USER_INPUT session status + instead (see ``task_exec._handle_need_user_input``). + """ key = f"linsight:task:owner:{session_version_id}" # Set the node ID as the owner of the task with a TTL await self.redis.aset(key, self.node_id, expiration=86400) # 1 day expiration - async def release_task_ownership(self, session_version_id): - """Release task ownership""" - key = f"linsight:task:owner:{session_version_id}" - await self.redis.adelete(key) - async def is_node_alive(self, target_node_id): """Check if a target node is alive based on its heartbeat""" if not target_node_id: diff --git a/src/backend/bisheng/main.py b/src/backend/bisheng/main.py index 77a8d29223..2bcfe26fbc 100644 --- a/src/backend/bisheng/main.py +++ b/src/backend/bisheng/main.py @@ -101,6 +101,15 @@ async def lifespan(app: FastAPI): await backfill_linsight_default_model() except Exception: logger.exception("linsight default-model backfill failed; continuing startup") + # Ships the kernel's built-in skills into every tenant so a fresh deploy + # has them without any operator step. Content-addressed and idempotent: + # an unchanged image costs a few file reads. + try: + from bisheng.linsight.domain.services.builtin_skill_seeder import seed_builtin_skills + + await seed_builtin_skills() + except Exception: + logger.exception("built-in linsight skill seeding failed; continuing startup") # LangfuseInstance.update() yield finally: diff --git a/src/backend/bisheng/permission/api/endpoints/grant_subjects.py b/src/backend/bisheng/permission/api/endpoints/grant_subjects.py new file mode 100644 index 0000000000..68140455f9 --- /dev/null +++ b/src/backend/bisheng/permission/api/endpoints/grant_subjects.py @@ -0,0 +1,174 @@ +"""Candidate subjects for granting permissions on one resource. + +"Who may I grant this resource to" is a different question from "which users do +I administer", and it needs a different predicate: holding `manage_permission` +**on the resource**. The pickers used to ask exactly this, through +`…/resources/{type}/{id}/grant-subjects/…`; F048 (`edcbe81b`) removed those +routes and pointed both frontends at the org-management endpoints instead, so a +knowledge-space manager who administers no department or user group saw an empty +user list and a permission error on the department tree. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query + +from bisheng.common.dependencies.user_deps import UserPayload +from bisheng.common.errcode.permission import PermissionDeniedError +from bisheng.common.schemas.api import UnifiedResponseModel, resp_200 +from bisheng.permission.api.responses import permission_error_response +from bisheng.permission.application.access import get_f048_resource_registry +from bisheng.permission.application.business_authorization import check_business_action +from bisheng.permission.application.identity import resolve_permission_actor +from bisheng.permission.domain.services import grant_subject_service +from bisheng.permission.domain.services.grant_subject_service import GrantSubjectScope + +router = APIRouter(tags=["Permission"]) + +GRANT_SUBJECT_RESOURCE_TYPES = frozenset( + { + "knowledge_space", + "knowledge_library", + "folder", + "knowledge_file", + "workflow", + "assistant", + "tool", + "channel", + "dashboard", + } +) + + +async def _authorized_scope( + resource_type: str, + resource_id: str, + login_user: UserPayload, +) -> GrantSubjectScope: + """Authorize the caller, then derive the candidate scope from the resource. + + The tenant comes from the verified target rather than from the caller, so a + super admin picking subjects for another tenant's resource still sees that + tenant's people. + """ + + actor = await resolve_permission_actor(login_user) + registry = await get_f048_resource_registry() + target = await registry.resolve( + resource_type=resource_type, + resource_id=resource_id, + actor=actor, + action="manage_permission", + ) + allowed = await check_business_action( + login_user, + resource_type=resource_type, + resource_id=resource_id, + action="manage_permission", + ) + if not allowed: + raise PermissionDeniedError() + return GrantSubjectScope( + tenant_id=int(target.tenant_id), + department_path=await grant_subject_service.resolve_department_space_path( + resource_type, + resource_id, + ), + ) + + +@router.get("/resources/{resource_type}/{resource_id}/grant-subjects/users") +async def list_grant_subject_users( + resource_type: str, + resource_id: str, + keyword: str = "", + page: int = Query(1, ge=1), + page_size: int = Query(50, ge=1, le=2000), + login_user: UserPayload = Depends(UserPayload.get_login_user), +) -> UnifiedResponseModel: + if resource_type not in GRANT_SUBJECT_RESOURCE_TYPES: + return permission_error_response(PermissionDeniedError()) + try: + scope = await _authorized_scope(resource_type, resource_id, login_user) + except PermissionDeniedError as error: + return permission_error_response(error) + rows = await grant_subject_service.list_candidate_users( + scope, + keyword=keyword, + page=page, + page_size=page_size, + ) + return resp_200({"data": rows, "total": len(rows)}) + + +@router.get("/resources/{resource_type}/{resource_id}/grant-subjects/user-groups") +async def list_grant_subject_user_groups( + resource_type: str, + resource_id: str, + keyword: str = "", + page: int = Query(1, ge=1), + page_size: int = Query(50, ge=1, le=2000), + login_user: UserPayload = Depends(UserPayload.get_login_user), +) -> UnifiedResponseModel: + if resource_type not in GRANT_SUBJECT_RESOURCE_TYPES: + return permission_error_response(PermissionDeniedError()) + try: + scope = await _authorized_scope(resource_type, resource_id, login_user) + except PermissionDeniedError as error: + return permission_error_response(error) + rows = await grant_subject_service.list_candidate_user_groups( + scope, + keyword=keyword, + page=page, + page_size=page_size, + ) + return resp_200({"data": rows, "total": len(rows)}) + + +@router.get("/resources/{resource_type}/{resource_id}/grant-subjects/departments/children") +async def list_grant_subject_department_children( + resource_type: str, + resource_id: str, + parent_id: int | None = None, + login_user: UserPayload = Depends(UserPayload.get_login_user), +) -> UnifiedResponseModel: + if resource_type not in GRANT_SUBJECT_RESOURCE_TYPES: + return permission_error_response(PermissionDeniedError()) + try: + scope = await _authorized_scope(resource_type, resource_id, login_user) + except PermissionDeniedError as error: + return permission_error_response(error) + return resp_200(await grant_subject_service.list_candidate_department_layer(scope, parent_id=parent_id)) + + +@router.get("/resources/{resource_type}/{resource_id}/grant-subjects/departments/search") +async def search_grant_subject_departments( + resource_type: str, + resource_id: str, + keyword: str = "", + limit: int = Query(200, ge=1, le=1000), + login_user: UserPayload = Depends(UserPayload.get_login_user), +) -> UnifiedResponseModel: + if resource_type not in GRANT_SUBJECT_RESOURCE_TYPES: + return permission_error_response(PermissionDeniedError()) + try: + scope = await _authorized_scope(resource_type, resource_id, login_user) + except PermissionDeniedError as error: + return permission_error_response(error) + return resp_200(await grant_subject_service.search_candidate_departments(scope, keyword=keyword, limit=limit)) + + +@router.get("/resources/{resource_type}/{resource_id}/grant-subjects/departments/{dept_id}/path-tree") +async def get_grant_subject_department_path_tree( + resource_type: str, + resource_id: str, + dept_id: int, + login_user: UserPayload = Depends(UserPayload.get_login_user), +) -> UnifiedResponseModel: + if resource_type not in GRANT_SUBJECT_RESOURCE_TYPES: + return permission_error_response(PermissionDeniedError()) + try: + scope = await _authorized_scope(resource_type, resource_id, login_user) + except PermissionDeniedError as error: + return permission_error_response(error) + return resp_200(await grant_subject_service.get_candidate_department_path(scope, dept_id=dept_id)) diff --git a/src/backend/bisheng/permission/api/router.py b/src/backend/bisheng/permission/api/router.py index 74cb7b2e29..2e79a9ab5b 100644 --- a/src/backend/bisheng/permission/api/router.py +++ b/src/backend/bisheng/permission/api/router.py @@ -5,8 +5,10 @@ from bisheng.permission.api.endpoints.catalog import router as catalog_router from bisheng.permission.api.endpoints.decision import router as decision_router from bisheng.permission.api.endpoints.grant import router as grant_router +from bisheng.permission.api.endpoints.grant_subjects import router as grant_subjects_router router = APIRouter(prefix='/permissions', tags=['Permission']) router.include_router(catalog_router) router.include_router(grant_router) router.include_router(decision_router) +router.include_router(grant_subjects_router) diff --git a/src/backend/bisheng/permission/application/business_authorization.py b/src/backend/bisheng/permission/application/business_authorization.py index fb862e0508..fab8e5281b 100644 --- a/src/backend/bisheng/permission/application/business_authorization.py +++ b/src/backend/bisheng/permission/application/business_authorization.py @@ -76,6 +76,13 @@ async def batch_check_business_actions( return {} actor = await resolve_permission_actor(login_user) + if actor.super_admin: + # The decision layer already allows a super admin unconditionally, but it + # only says so after every candidate has been resolved — several queries + # each, per action. Listing a page of 100 candidates against 5 actions + # therefore paid 500 resolutions to reach a foregone conclusion. + return {resource_id: frozenset(normalized_actions) for resource_id in normalized_ids} + registry = await get_f048_resource_registry() result: dict[str, set[str]] = {resource_id: set() for resource_id in normalized_ids} runtime = await get_f048_runtime() diff --git a/src/backend/bisheng/permission/application/catalog_api.py b/src/backend/bisheng/permission/application/catalog_api.py index f9907d9406..d742bb5eb9 100644 --- a/src/backend/bisheng/permission/application/catalog_api.py +++ b/src/backend/bisheng/permission/application/catalog_api.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio from collections.abc import AsyncIterator, Callable from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, replace @@ -78,6 +79,7 @@ ZERO_CHECKSUM = "0" * 64 HIGHER_CONSISTENCY = "HIGHER_CONSISTENCY" CATALOG_STAGE_BATCH_SIZE = 80 +CATALOG_READ_CONCURRENCY = 8 ACTIVE_OPERATION_STATUSES = ( "PREPARED", "STAGING", @@ -1087,10 +1089,7 @@ async def stage_model_releases( ) -> None: expected = self._expected_tuples(draft) await self._persist_plan(draft.release_id, expected) - present = { - (row["user"], row["relation"], row["object"]) - for row in await self._client.read_tuples(consistency=HIGHER_CONSISTENCY) - } + present = await self._read_present(expected) missing = [row for row in expected if (row["user"], row["relation"], row["object"]) not in present] for index in range(0, len(missing), CATALOG_STAGE_BATCH_SIZE): batch = missing[index : index + CATALOG_STAGE_BATCH_SIZE] @@ -1102,17 +1101,43 @@ async def run_model_tests( self, draft: CatalogDraftSnapshot, ) -> None: - expected = {(row["user"], row["relation"], row["object"]) for row in self._expected_tuples(draft)} - present = { - (row["user"], row["relation"], row["object"]) - for row in await self._client.read_tuples(consistency=HIGHER_CONSISTENCY) - } + planned = self._expected_tuples(draft) + expected = {(row["user"], row["relation"], row["object"]) for row in planned} + present = await self._read_present(planned) missing = expected - present if missing: raise PermissionProjectionFailedError(msg=f"Catalog staged tuple verification failed: {len(missing)}") if draft.model_release is None: raise PermissionPublishNotReadyError(msg="Catalog model release is missing") + async def _read_present( + self, + planned: list[dict[str, str]], + ) -> set[tuple[str, str, str]]: + """Read only the tuples this plan touches, one concrete object at a time. + + An unfiltered Read walks the whole Store at 100 tuples per request, so + checking a few hundred Catalog tuples against a 77k-tuple Store cost + ~774 round trips — twice per publish, at HIGHER_CONSISTENCY. Scoping the + reads to the planned objects makes the cost proportional to the plan + instead of the Store, and the objects are independent so they overlap. + """ + + objects = sorted({row["object"] for row in planned}) + if not objects: + return set() + semaphore = asyncio.Semaphore(CATALOG_READ_CONCURRENCY) + + async def read(object_key: str) -> list[dict]: + async with semaphore: + return await self._client.read_tuples( + object=object_key, + consistency=HIGHER_CONSISTENCY, + ) + + pages = await asyncio.gather(*(read(key) for key in objects)) + return {(row["user"], row["relation"], row["object"]) for page in pages for row in page} + async def arm_recent_marker( self, draft: CatalogDraftSnapshot, @@ -1149,9 +1174,12 @@ async def commit_active( return _commit_checksum(changes) async def read_active_release_keys(self) -> frozenset[str]: + # OpenFGA rejects a tuple_key without an object type, so the filter has + # to name the type even though the prefix check below already does. rows = await self._client.read_tuples( user="user:*", relation="active", + object="permission_catalog_release:", consistency=HIGHER_CONSISTENCY, ) prefix = "permission_catalog_release:" @@ -1320,8 +1348,8 @@ async def create_draft( before = await self._state.load_snapshot(reservation.predecessor_id) if before.action_release is None or before.model_release is None: raise PermissionPublishNotReadyError() - actions, customs, standard_policy = await self._apply_change( - request.change, + actions, customs, standard_policy = await self._apply_changes( + request.changes, before, ) try: @@ -1454,15 +1482,22 @@ async def _release_payload( "published_at": (_as_utc(row.published_at).isoformat() if row.published_at is not None else None), } - async def _apply_change( + async def _apply_changes( self, - change: CatalogChangeRequest, + changes: tuple[CatalogChangeRequest, ...], before: CatalogDraftSnapshot, ) -> tuple[ tuple[CatalogAction, ...], tuple[CustomModelSelection, ...], dict[str, bool], ]: + """Fold the whole edit batch onto the base release, then validate once. + + Validation runs on the resulting state rather than after each change: a + batch may pass through an intermediate arrangement it never publishes, + and only the state that actually ships has to hold. + """ + assert before.action_release is not None assert before.model_release is not None actions = list(before.action_release.actions) @@ -1480,90 +1515,99 @@ async def _apply_change( } standard_by_key = {model.model_key: model for model in before.model_release.models if model.kind == "STANDARD"} standard_policy = {key: model.allow_same_level for key, model in standard_by_key.items()} - kind = change.type - if kind in { - CatalogChangeType.ASSIGN_ACTION_LEVEL, - CatalogChangeType.SET_ACTION_ACTIVE, - }: - index = next( - (index for index, action in enumerate(actions) if action.code == change.action_code), - None, - ) - if index is None: - raise InvalidCatalogActionError() - if kind == CatalogChangeType.ASSIGN_ACTION_LEVEL: - actions[index] = replace( - actions[index], - level=(int(change.level) if change.level is not None else None), - ) - elif change.active is None: - raise InvalidCatalogActionError() - else: - actions[index] = replace( - actions[index], - active=change.active, + touched_standard_keys: set[str] = set() + for change in changes: + kind = change.type + if kind in { + CatalogChangeType.ASSIGN_ACTION_LEVEL, + CatalogChangeType.SET_ACTION_ACTIVE, + }: + index = next( + (index for index, action in enumerate(actions) if action.code == change.action_code), + None, ) - elif kind == CatalogChangeType.CREATE_MODEL: - key = change.model_key or uuid4().hex - if key in custom_by_key or key in standard_by_key or not change.name or not change.action_codes: - raise PermissionModelStateConflictError() - custom_by_key[key] = CustomModelSelection( - model_key=key, - name=change.name, - action_codes=change.action_codes, - active=change.active is not False, - allow_same_level=bool(change.allow_same_level), - ) - elif kind == CatalogChangeType.UPDATE_MODEL: - model = self._custom_model(change.model_key, custom_by_key) - custom_by_key[model.model_key] = replace( - model, - name=change.name if change.name is not None else model.name, - action_codes=(change.action_codes if change.action_codes is not None else model.action_codes), - active=(change.active if change.active is not None else model.active), - allow_same_level=( - change.allow_same_level if change.allow_same_level is not None else model.allow_same_level - ), - ) - elif kind == CatalogChangeType.SET_MODEL_ACTIVE: - if change.active is None: - raise PermissionModelStateConflictError() - model = self._custom_model(change.model_key, custom_by_key) - custom_by_key[model.model_key] = replace( - model, - active=change.active, - ) - elif kind == CatalogChangeType.DELETE_MODEL: - model = self._custom_model(change.model_key, custom_by_key) - derived = next(item for item in before.model_release.models if item.model_key == model.model_key) - references = await self._state.grant_references() - try: - ensure_model_deletable( - derived, - reference_count=len(references.get(model.model_key, ())), + if index is None: + raise InvalidCatalogActionError() + if kind == CatalogChangeType.ASSIGN_ACTION_LEVEL: + actions[index] = replace( + actions[index], + level=(int(change.level) if change.level is not None else None), + ) + elif change.active is None: + raise InvalidCatalogActionError() + else: + actions[index] = replace( + actions[index], + active=change.active, + ) + elif kind == CatalogChangeType.CREATE_MODEL: + key = change.model_key or uuid4().hex + if key in custom_by_key or key in standard_by_key or not change.name or not change.action_codes: + raise PermissionModelStateConflictError() + custom_by_key[key] = CustomModelSelection( + model_key=key, + name=change.name, + action_codes=change.action_codes, + active=change.active is not False, + allow_same_level=bool(change.allow_same_level), ) - except ValueError as exc: - raise PermissionModelStateConflictError( - exception=exc, - msg=str(exc), - ) from exc - del custom_by_key[model.model_key] - elif kind == CatalogChangeType.SET_ALLOW_SAME_LEVEL: - if change.allow_same_level is None or not change.model_key: - raise PermissionModelStateConflictError() - if change.model_key in standard_by_key: - standard_policy[change.model_key] = change.allow_same_level - else: - model = self._custom_model( - change.model_key, - custom_by_key, + elif kind == CatalogChangeType.UPDATE_MODEL: + model = self._custom_model(change.model_key, custom_by_key) + custom_by_key[model.model_key] = replace( + model, + name=change.name if change.name is not None else model.name, + action_codes=(change.action_codes if change.action_codes is not None else model.action_codes), + active=(change.active if change.active is not None else model.active), + allow_same_level=( + change.allow_same_level if change.allow_same_level is not None else model.allow_same_level + ), ) + elif kind == CatalogChangeType.SET_MODEL_ACTIVE: + if change.active is None: + raise PermissionModelStateConflictError() + model = self._custom_model(change.model_key, custom_by_key) custom_by_key[model.model_key] = replace( model, - allow_same_level=change.allow_same_level, + active=change.active, ) - else: - raise InvalidCatalogActionError() + elif kind == CatalogChangeType.DELETE_MODEL: + model = self._custom_model(change.model_key, custom_by_key) + derived = next(item for item in before.model_release.models if item.model_key == model.model_key) + references = await self._state.grant_references() + try: + # Judge "is it disabled" on the state this batch publishes, not + # on the base release. Reading the base meant a batch that + # deactivates and then deletes was refused for being active, + # forcing two separate publications to remove one model. + ensure_model_deletable( + replace(derived, active=model.active), + reference_count=len(references.get(model.model_key, ())), + ) + except ValueError as exc: + raise PermissionModelStateConflictError( + exception=exc, + msg=str(exc), + ) from exc + del custom_by_key[model.model_key] + elif kind == CatalogChangeType.SET_ALLOW_SAME_LEVEL: + if change.allow_same_level is None or not change.model_key: + raise PermissionModelStateConflictError() + if change.model_key in standard_by_key: + standard_policy[change.model_key] = change.allow_same_level + else: + model = self._custom_model( + change.model_key, + custom_by_key, + ) + custom_by_key[model.model_key] = replace( + model, + allow_same_level=change.allow_same_level, + ) + else: + raise InvalidCatalogActionError() + if change.model_key and change.model_key in standard_by_key: + if change.type is not CatalogChangeType.SET_ALLOW_SAME_LEVEL: + touched_standard_keys.add(change.model_key) try: action_release = derive_action_release(actions) derive_permission_models( @@ -1572,7 +1616,7 @@ async def _apply_change( standard_allow_same_level=standard_policy, ) except ValueError as exc: - if change.model_key in standard_by_key and kind not in {CatalogChangeType.SET_ALLOW_SAME_LEVEL}: + if touched_standard_keys: raise ImmutableStandardModelError( exception=exc, msg=str(exc), diff --git a/src/backend/bisheng/permission/application/control_state.py b/src/backend/bisheng/permission/application/control_state.py index ee15d64cfb..6f110511c5 100644 --- a/src/backend/bisheng/permission/application/control_state.py +++ b/src/backend/bisheng/permission/application/control_state.py @@ -372,6 +372,12 @@ async def load_source_page( limit=fetch_limit, protected_only=False, ) + # The parent's creator carries no authority here — this resource + # has its own protected creator row, and the inherited copy only + # showed up as a second, identical entry that cannot be acted on. + inherited_rows = [ + (row, model_key) for row, model_key in inherited_rows if row.source_type != "CREATOR" + ] combined = sorted( ( *((row, model_key, "LOCAL") for row, model_key in local_rows), diff --git a/src/backend/bisheng/permission/application/process_runtime.py b/src/backend/bisheng/permission/application/process_runtime.py index 08fa1a0637..2f640fe04d 100644 --- a/src/backend/bisheng/permission/application/process_runtime.py +++ b/src/backend/bisheng/permission/application/process_runtime.py @@ -38,7 +38,7 @@ async def bind_catalog_runtime( async def heartbeat(self) -> bool: ... - async def mark_migration_required(self) -> None: ... + async def mark_migration_required(self, *, reason: str = ...) -> None: ... def readiness(self) -> dict: ... @@ -78,15 +78,21 @@ def register_f048_permission_runtime_context( async def initialize() -> ProcessPermissionRuntime: manager = app_context.get_context("openfga") client = await manager.async_get_instance() - if manager.readiness().get("migration_required"): - raise PermissionPublishNotReadyError(msg="Permission data migration is required") + readiness = manager.readiness() + if readiness.get("migration_required"): + # Report what actually latched the gate. A fenced Catalog left by a + # crashed publish reported "migration is required" here, which sent + # the operator looking for a migration that was never pending. + raise PermissionPublishNotReadyError( + msg=str(readiness.get("error") or "Permission data migration is required") + ) try: runtime = await initializer(client) except ( AuthorizationModelMismatchError, PermissionPublishNotReadyError, - ): - await manager.mark_migration_required() + ) as exc: + await manager.mark_migration_required(reason=str(exc) or "permission_data_migration_required") raise await bind_f048_process_runtime( manager, diff --git a/src/backend/bisheng/permission/application/resource_api.py b/src/backend/bisheng/permission/application/resource_api.py index 270cf56876..bcfde7eeb6 100644 --- a/src/backend/bisheng/permission/application/resource_api.py +++ b/src/backend/bisheng/permission/application/resource_api.py @@ -58,6 +58,18 @@ async def display_names( subjects: tuple[tuple[str, str], ...], ) -> dict[tuple[str, str], str]: ... + async def resource_display_names( + self, + resources: tuple[tuple[str, str], ...], + ) -> dict[tuple[str, str], str]: ... + + + +def _split_resource_key(resource_key: str) -> tuple[str, str]: + """Split "knowledge_space:3377" into its type and id.""" + + resource_type, _, resource_id = resource_key.partition(":") + return resource_type, resource_id def _encode_cursor(payload: dict[str, object]) -> str: raw = json.dumps( @@ -180,10 +192,19 @@ async def list_grants( names = await self._subjects.display_names( tuple(dict.fromkeys((row.subject_type, row.subject_id) for row in selected)) ) + # The permission layer holds a resource's identity, never its label — the + # roster used to render "knowledge_space:3377" at users. Resolved through + # the business side, the same way subject names already are. + parents = tuple( + dict.fromkeys( + _split_resource_key(row.inherited_from) for row in selected if row.inherited_from + ) + ) + parent_names = await self._subjects.resource_display_names(parents) if parents else {} model_names = {item.snapshot.model_key: item.name for item in catalog.models} data = [ { - "assignee_id": row.source_id, + "assignee_id": str(row.source_id), "assignee_version": row.source_version, "subject": { "type": row.subject_type, @@ -205,6 +226,9 @@ async def list_grants( }, "scope": row.scope, "inherited_from": row.inherited_from, + "inherited_from_name": ( + parent_names.get(_split_resource_key(row.inherited_from)) if row.inherited_from else None + ), "protected": row.protected, "editable": row.editable, } @@ -294,7 +318,7 @@ async def mutate_grants( operation=change.op.value, model_key=change.model_key, source=source, - assignee_id=change.assignee_id, + assignee_id=change.assignee_row_id, expected_assignee_version=(change.expected_assignee_version), target_model_key=change.target_model_key, ) @@ -309,7 +333,7 @@ async def mutate_grants( ) items = [ { - "assignee_id": source.source_id, + "assignee_id": str(source.source_id), "assignee_version": source.version, "subject": { "type": source.subject_type, @@ -370,7 +394,10 @@ async def create_mode_draft( "draft_id": draft.draft_id, "target_mode": draft.target_mode, "impact_checksum": draft.impact_checksum, - "affected_assignees": len(draft.snapshot_sources), + # Either direction disturbs people: CUSTOM copies the inherited members + # down, INHERIT drops the local ones. Counting only the copies reported + # zero for a switch that was about to remove grants. + "affected_assignees": (len(draft.snapshot_sources) + len(draft.discarded_sources)), "expires_at": (datetime.now(UTC) + timedelta(minutes=10)).isoformat(), } diff --git a/src/backend/bisheng/permission/domain/schemas/f048.py b/src/backend/bisheng/permission/domain/schemas/f048.py index 90172d0eba..6225874a7e 100644 --- a/src/backend/bisheng/permission/domain/schemas/f048.py +++ b/src/backend/bisheng/permission/domain/schemas/f048.py @@ -186,7 +186,10 @@ class CatalogChangeRequest(StrictRequestModel): class CatalogDraftRequest(StrictRequestModel): idempotency_key: str = Field(min_length=1, max_length=64) base_release_id: int = Field(gt=0) - change: CatalogChangeRequest + # A draft carries the whole edit batch. One change per draft forced the UI to + # open a fresh draft off the CURRENT release for every edit, so publishing + # applied only the last one and silently dropped the rest. + changes: tuple[CatalogChangeRequest, ...] = Field(min_length=1, max_length=50) class CatalogImpactDTO(BaseModel): @@ -225,10 +228,16 @@ class GrantMutationChange(StrictRequestModel): op: GrantMutationOperation model_key: str | None = Field(default=None, max_length=64) subject: GrantSubjectInput | None = None - assignee_id: int | None = Field(default=None, gt=0) + # Carried as a decimal string: row ids are 60-62 bit, past the 2^53 that a + # JSON number survives in a browser, and a rounded id matches no assignee. + assignee_id: str | None = Field(default=None, pattern=r"^[1-9][0-9]{0,19}$") expected_assignee_version: int | None = Field(default=None, ge=0) target_model_key: str | None = Field(default=None, max_length=64) + @property + def assignee_row_id(self) -> int | None: + return None if self.assignee_id is None else int(self.assignee_id) + @model_validator(mode="after") def validate_operation_shape(self) -> GrantMutationChange: if self.op == GrantMutationOperation.ADD: @@ -279,13 +288,14 @@ class GrantSourceDTO(BaseModel): class GrantAssigneeDTO(BaseModel): - assignee_id: int + assignee_id: str assignee_version: int subject: GrantSubjectDTO model: GrantModelDTO source: GrantSourceDTO scope: Literal["LOCAL", "INHERITED"] inherited_from: str | None = None + inherited_from_name: str | None = None protected: bool editable: bool diff --git a/src/backend/bisheng/permission/domain/services/catalog_service.py b/src/backend/bisheng/permission/domain/services/catalog_service.py index 0da63e1bc1..6cee2fcfb0 100644 --- a/src/backend/bisheng/permission/domain/services/catalog_service.py +++ b/src/backend/bisheng/permission/domain/services/catalog_service.py @@ -437,7 +437,21 @@ async def _resolve_unknown_commit( original_error: Exception, allow_retry: bool, ) -> CatalogPublishOutcome: - active = await self._projector.read_active_release_keys() + try: + active = await self._projector.read_active_release_keys() + except Exception as exc: + # Reading the pointer is how this path tells "committed" from "never + # committed". If that read itself fails the publication is + # unresolvable, so land it in the fenced terminal state instead of + # letting the error escape — an escaping error leaves the CURRENT + # release fenced with no FAILED_CLOSED marker and no event, which is + # invisible until a restart takes the whole permission runtime down. + return await self._fail_closed( + context, + reason=( + f"Catalog active pointer is unreadable after commit: error={exc}, original_error={original_error}" + ), + ) old_key = context.current_release_key new_key = context.draft.release_key @@ -467,9 +481,29 @@ async def _resolve_unknown_commit( reconciled=True, ) - reason = ( - f"Catalog active pointer invariant violated after commit: active={sorted(active)}, error={original_error}" + await self._fail_closed( + context, + reason=( + f"Catalog active pointer invariant violated after commit: " + f"active={sorted(active)}, error={original_error}" + ), ) + + async def _fail_closed( + self, + context: CatalogPublishContext, + *, + reason: str, + ) -> CatalogPublishOutcome: + """Record the fenced terminal state, then raise. + + Publishing fences the CURRENT release and only a resolved commit lifts + that fence, so an unresolvable publication is meant to stay fenced. What + it must never do is stay fenced *unlabelled*: the release has to end up + FAILED_CLOSED with a reason, or the next process restart refuses to serve + permissions with nothing on record explaining why. + """ + await self._state.fail_closed(context, reason=reason) await self._emit( context, diff --git a/src/backend/bisheng/permission/domain/services/grant_subject_service.py b/src/backend/bisheng/permission/domain/services/grant_subject_service.py new file mode 100644 index 0000000000..9c5526f4e4 --- /dev/null +++ b/src/backend/bisheng/permission/domain/services/grant_subject_service.py @@ -0,0 +1,257 @@ +"""Candidate subjects for granting one resource: users, groups, departments. + +Answering "who may I grant this resource to" needs a different predicate from +"which users do I administer" — holding `manage_permission` **on the resource**, +not membership of an organisational admin role. The pickers used to ask exactly +this; F048 pointed them at the org-management endpoints instead, so a knowledge +space's manager who administers no department saw an empty user list and a +permission error on the department tree. + +The queries here carry over from that earlier implementation, including the +department-space narrowing (F033) and the prefix keyword match that keeps the +`user_name` index usable (F038). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlmodel import col, select + +from bisheng.core.context.tenant import bypass_tenant_filter +from bisheng.core.database import get_async_db_session +from bisheng.database.models.department import Department, DepartmentDao, UserDepartment +from bisheng.database.models.group import Group +from bisheng.database.models.tenant import UserTenant +from bisheng.department.domain.services.department_service import ( + DepartmentService, + _dept_node_dict, +) +from bisheng.knowledge.domain.models.department_knowledge_space import ( + DepartmentKnowledgeSpaceDao, +) +from bisheng.user.domain.models.user import User + +# A department-bound space whose department is gone has no candidate in scope; +# an empty path would match everything instead of nothing. +_MATCHES_NOTHING = "\x00never-matches" + + +@dataclass(frozen=True, slots=True) +class GrantSubjectScope: + """The resource's tenant, plus the department subtree it is confined to.""" + + tenant_id: int + department_path: str | None + + +async def resolve_department_space_path(resource_type: str, resource_id: str) -> str | None: + """Narrow a department-bound knowledge space to that department's subtree. + + Derived from the binding, never from anything the client sends, so a direct + API call cannot widen the candidate set (F033). + """ + + if resource_type != "knowledge_space": + return None + try: + space_id = int(resource_id) + except (TypeError, ValueError): + return None + binding = await DepartmentKnowledgeSpaceDao.aget_by_space_id(space_id) + if binding is None: + return None + department = await DepartmentDao.aget_by_id(int(binding.department_id)) + if department is None or getattr(department, "status", "active") != "active": + return _MATCHES_NOTHING + return str(department.path) + + +async def list_candidate_users( + scope: GrantSubjectScope, + *, + keyword: str, + page: int, + page_size: int, +) -> list[dict]: + with bypass_tenant_filter(): + async with get_async_db_session() as session: + in_tenant = ( + select(UserTenant.id) + .where( + UserTenant.user_id == User.user_id, + UserTenant.tenant_id == scope.tenant_id, + UserTenant.status == "active", + ) + .exists() + ) + statement = ( + select(User.user_id, User.user_name, User.external_id) + .where(User.delete == 0, in_tenant) + .order_by(col(User.user_id).desc()) + ) + if scope.department_path is not None: + in_subtree = ( + select(UserDepartment.id) + .join(Department, Department.id == UserDepartment.department_id) + .where( + UserDepartment.user_id == User.user_id, + col(Department.path).like(f"{scope.department_path}%"), + Department.status == "active", + ) + .exists() + ) + statement = statement.where(in_subtree) + if keyword: + # Prefix match keeps the user_name index usable; a leading + # wildcard forced a full scan of a 150k-row table (F038). + statement = statement.where(col(User.user_name).like(f"{keyword}%")) + rows = (await session.exec(statement.offset((page - 1) * page_size).limit(page_size))).all() + # The picker prints the staff id after the name and the department under it, + # so people with the same display name can be told apart. + paths = await _primary_department_paths([int(row.user_id) for row in rows]) + return [ + { + "user_id": int(row.user_id), + "user_name": row.user_name, + "external_id": row.external_id, + "primary_department_path": paths.get(int(row.user_id)), + } + for row in rows + ] + + +async def _primary_department_paths(user_ids: list[int]) -> dict[int, str]: + """Each user's primary department as a readable name chain, in one round trip.""" + + if not user_ids: + return {} + with bypass_tenant_filter(): + async with get_async_db_session() as session: + rows = ( + await session.exec( + select(UserDepartment.user_id, Department.path) + .join(Department, Department.id == UserDepartment.department_id) + .where( + col(UserDepartment.user_id).in_(user_ids), + UserDepartment.is_primary == 1, + Department.status == "active", + ) + ) + ).all() + if not rows: + return {} + needed: set[int] = set() + for row in rows: + needed.update(int(part) for part in str(row.path).strip("/").split("/") if part.isdigit()) + names = {int(d.id): d.name for d in await DepartmentDao.aget_by_ids(list(needed)) if d.id is not None} + resolved: dict[int, str] = {} + for row in rows: + chain = [ + names[int(part)] for part in str(row.path).strip("/").split("/") if part.isdigit() and int(part) in names + ] + if chain: + resolved[int(row.user_id)] = "/".join(chain) + return resolved + + +async def list_candidate_user_groups( + scope: GrantSubjectScope, + *, + keyword: str, + page: int, + page_size: int, +) -> list[dict]: + with bypass_tenant_filter(): + async with get_async_db_session() as session: + statement = select(Group.id, Group.group_name).where(Group.tenant_id == scope.tenant_id) + if keyword: + statement = statement.where(col(Group.group_name).like(f"{keyword}%")) + statement = statement.order_by(col(Group.id).desc()).offset((page - 1) * page_size).limit(page_size) + rows = (await session.exec(statement)).all() + return [{"id": int(row.id), "name": row.group_name} for row in rows] + + +async def list_candidate_department_layer( + scope: GrantSubjectScope, + *, + parent_id: int | None, +) -> list[dict]: + """One layer of the department tree, so a large organisation never loads at once.""" + + with bypass_tenant_filter(): + async with get_async_db_session() as session: + statement = select(Department).where( + Department.tenant_id == scope.tenant_id, + Department.status == "active", + ) + if parent_id is None: + # A department-bound space starts at its own department, not at + # the tenant root. + if scope.department_path is not None: + statement = statement.where(col(Department.path) == scope.department_path) + else: + statement = statement.where(col(Department.parent_id).is_(None)) + else: + statement = statement.where(Department.parent_id == parent_id) + if scope.department_path is not None: + statement = statement.where(col(Department.path).like(f"{scope.department_path}%")) + rows = (await session.exec(statement.order_by(col(Department.id)))).all() + return await _as_tree_nodes(rows) + + +async def search_candidate_departments( + scope: GrantSubjectScope, + *, + keyword: str, + limit: int, +) -> dict: + if not keyword: + return {"roots": [], "total_matches": 0, "truncated": False} + with bypass_tenant_filter(): + async with get_async_db_session() as session: + statement = select(Department).where( + Department.tenant_id == scope.tenant_id, + Department.status == "active", + col(Department.name).like(f"%{keyword}%"), + ) + if scope.department_path is not None: + statement = statement.where(col(Department.path).like(f"{scope.department_path}%")) + rows = (await session.exec(statement.order_by(col(Department.id)).limit(limit + 1))).all() + truncated = len(rows) > limit + matches = list(rows[:limit]) + roots = await DepartmentService.abuild_forest_within_subtree( + matches, + {int(row.id) for row in matches if row.id is not None}, + confined_to_path=None if scope.department_path is None else scope.department_path, + ) + return {"roots": roots, "total_matches": len(matches), "truncated": truncated} + + +async def get_candidate_department_path(scope: GrantSubjectScope, *, dept_id: int) -> dict: + """Reveal one department: the pruned tree from the root down to it. + + Same envelope as the org-management locate endpoint, because the picker's + tree renders the two interchangeably. + """ + + empty = {"roots": [], "total_matches": 0, "truncated": False} + department = await DepartmentDao.aget_by_id(dept_id) + if department is None or int(department.tenant_id) != scope.tenant_id: + return empty + if scope.department_path is not None and not str(department.path).startswith(scope.department_path): + return empty + roots = await DepartmentService.abuild_forest_within_subtree( + [department], + {int(department.id)}, + confined_to_path=scope.department_path, + ) + return {"roots": roots, "total_matches": 1, "truncated": False} + + +async def _as_tree_nodes(rows: list) -> list[dict]: + """One rendered layer, in the org tree's node shape (`has_children` batched).""" + + ids = [int(row.id) for row in rows if row.id is not None] + with_children = await DepartmentDao.aget_children_existence(ids) if ids else set() + return [_dept_node_dict(row, has_children=int(row.id) in with_children) for row in rows] diff --git a/src/backend/bisheng/permission/domain/services/mode_service.py b/src/backend/bisheng/permission/domain/services/mode_service.py index 4abae85609..1e32efa871 100644 --- a/src/backend/bisheng/permission/domain/services/mode_service.py +++ b/src/backend/bisheng/permission/domain/services/mode_service.py @@ -69,6 +69,11 @@ class PermissionModeDraft: snapshot_sources: tuple[GrantSourceRecord, ...] staging_deltas: tuple[ProjectionTupleDelta, ...] result_grants: tuple[GrantSnapshot, ...] + # Ordinary local sources this switch discards. Switching to INHERIT drops + # every non-protected local grant, but only the CUSTOM direction produced + # snapshot_sources, so the confirmation reported "0 affected" while it was + # about to remove people. + discarded_sources: tuple[GrantSourceRecord, ...] = () @dataclass(frozen=True, slots=True) @@ -168,12 +173,19 @@ async def create_draft( ) -> PermissionModeDraft: target_mode = target_mode.upper() self._validate_switch(context, target_mode) + discarded_sources: tuple[GrantSourceRecord, ...] = () if target_mode == "CUSTOM": result_grants, snapshot_sources, staging = await self._snapshot_inherited(context) else: result_grants = self._protected_only(context.local_grants) snapshot_sources = () staging = () + discarded_sources = tuple( + source + for grant in context.local_grants + for source in grant.sources + if source.active and not source.protected + ) context_checksum = self._context_checksum(context) impact_payload = { @@ -199,6 +211,7 @@ async def create_draft( context_checksum=context_checksum, impact_checksum=impact_checksum, snapshot_sources=snapshot_sources, + discarded_sources=discarded_sources, staging_deltas=staging, result_grants=result_grants, ) diff --git a/src/backend/bisheng/tenant/domain/services/f048_permission_subject.py b/src/backend/bisheng/tenant/domain/services/f048_permission_subject.py index de71bcf204..150feb597e 100644 --- a/src/backend/bisheng/tenant/domain/services/f048_permission_subject.py +++ b/src/backend/bisheng/tenant/domain/services/f048_permission_subject.py @@ -115,3 +115,34 @@ async def display_names( **{("department", str(row.id)): row.name for row in departments if row.id is not None}, **{("user_group", str(row.id)): row.group_name for row in groups if row.id is not None}, } + + async def resource_display_names( + self, + resources: tuple[tuple[str, str], ...], + ) -> dict[tuple[str, str], str]: + """Label the resources a grant can be inherited from. + + The permission layer knows a resource's identity, never its name, so the + roster reported inheritance as "knowledge_space:3377". Only the container + types can be a permission parent, and both live in the knowledge table. + Anything else resolves to nothing and the caller keeps showing the id. + """ + + from bisheng.knowledge.domain.models.knowledge import KnowledgeDao + + knowledge_ids = [ + int(resource_id) + for resource_type, resource_id in resources + if resource_type in {"knowledge_space", "knowledge_library"} and resource_id.isdigit() + ] + if not knowledge_ids: + return {} + rows = await KnowledgeDao.aget_list_by_ids(knowledge_ids) + by_id = {int(row.id): row.name for row in rows or () if row.id is not None} + return { + (resource_type, resource_id): by_id[int(resource_id)] + for resource_type, resource_id in resources + if resource_type in {"knowledge_space", "knowledge_library"} + and resource_id.isdigit() + and int(resource_id) in by_id + } diff --git a/src/backend/bisheng/tenant/domain/services/tenant_service.py b/src/backend/bisheng/tenant/domain/services/tenant_service.py index 4bfebd6ff6..247f4eaac2 100644 --- a/src/backend/bisheng/tenant/domain/services/tenant_service.py +++ b/src/backend/bisheng/tenant/domain/services/tenant_service.py @@ -137,6 +137,17 @@ async def acreate_tenant(cls, data: TenantCreate, login_user) -> dict: await WorkStationService.acopy_root_builtin_tools_to_tenant(tenant.id) + # Step 6: seed the kernel's built-in Linsight skills. Startup seeding + # only covers tenants that existed then, so without this a tenant + # created later would never get them. Best-effort: a skill is not + # worth failing tenant creation over, and the next restart re-seeds. + try: + from bisheng.linsight.domain.services.builtin_skill_seeder import seed_builtin_skills + + await seed_builtin_skills([tenant.id]) + except Exception: + logger.warning("built-in skill seeding failed for new tenant %s", tenant.id, exc_info=True) + return _safe_tenant_dump(tenant) except TenantCodeDuplicateError: diff --git a/src/backend/bisheng/user/api/user.py b/src/backend/bisheng/user/api/user.py index 82763e2f2b..a235cf3788 100644 --- a/src/backend/bisheng/user/api/user.py +++ b/src/backend/bisheng/user/api/user.py @@ -453,9 +453,11 @@ async def list_user( org_scoped_ids.update(tenant_scoped_ids) if not managed_groups: - # 仅部门 / 子租户管理员：仅能看其管辖范围内用户 - if org_scoped_ids is None: - raise HTTPException(status_code=500, detail="Quit that! You don't have rights to view this.") + # 仅部门 / 子租户管理员：仅能看其管辖范围内用户。 + # 「没有组织管辖范围」和「管辖范围为空」对调用方是同一件事——通过这个 + # 接口你看不到任何人——所以同样返回空页，而不是 500。原先前者报错， + # 让持有资源 manage_permission 但不属于任何组织管理员角色的用户 + # （例如知识空间管理员）在授权选人时直接拿到 500，对话框整个用不了。 if not org_scoped_ids: return resp_200({"data": [], "total": 0}) user_ids = list(org_scoped_ids) diff --git a/src/backend/bisheng/worker/knowledge/rebuild_knowledge_worker.py b/src/backend/bisheng/worker/knowledge/rebuild_knowledge_worker.py index 22066cdf9d..d4f2ced517 100644 --- a/src/backend/bisheng/worker/knowledge/rebuild_knowledge_worker.py +++ b/src/backend/bisheng/worker/knowledge/rebuild_knowledge_worker.py @@ -68,28 +68,33 @@ def rebuild_knowledge_celery(knowledge_id: int, new_model_id: int, invoke_user_i file.remark = KnowledgeFileFailedError(data={"exception": "rebuild error"}).to_json_str() KnowledgeFileDao.update(file) - # 5. Update knowledge base status + # 5. Release the container; the failure lives on the files that failed. + # Marking the whole knowledge FAILED existed to drive a later automatic + # rebuild, which no longer happens — a failed file is recovered by + # re-parsing it. All that flag does now is take the entire knowledge + # down: permission targets require PUBLISHED, so a single bad file made + # the space unreachable and reported it as "invalid resource type or ID". if failed_files: # DeleteesIndex andmilvusCollections to avoid data inconsistencies _delete_es_files(knowledge, failed_files) - - knowledge.state = KnowledgeState.FAILED.value logger.error(f"knowledge_id={knowledge_id} rebuild failed, failed_files={failed_files}") else: - knowledge.state = KnowledgeState.PUBLISHED.value logger.info(f"knowledge_id={knowledge_id} rebuild completed successfully") + knowledge.state = KnowledgeState.PUBLISHED.value KnowledgeDao.update_one(knowledge) return f"knowledge {knowledge_id} rebuild completed" except Exception as e: logger.exception(f"rebuild_knowledge_celery error: {e!s}") - # Unexpected handles during asynchronous tasksknowledgeSet to4 + # Leave the knowledge usable rather than stranding it in REBUILDING: the + # run is over either way, and the per-file status already records what + # did not make it. try: knowledge = KnowledgeDao.query_by_id(knowledge_id) if knowledge: - knowledge.state = KnowledgeState.FAILED.value + knowledge.state = KnowledgeState.PUBLISHED.value KnowledgeDao.update_one(knowledge) except Exception as e2: logger.exception(f"Failed to update knowledge state after error: {e2!s}") diff --git a/src/backend/bisheng_langchain/gpts/tools/code_interpreter/base_executor.py b/src/backend/bisheng_langchain/gpts/tools/code_interpreter/base_executor.py index 3b7bcbc602..eff2bd1ebc 100644 --- a/src/backend/bisheng_langchain/gpts/tools/code_interpreter/base_executor.py +++ b/src/backend/bisheng_langchain/gpts/tools/code_interpreter/base_executor.py @@ -1,3 +1,4 @@ +import os import re from abc import ABC, abstractmethod from datetime import timedelta @@ -104,6 +105,10 @@ def __init__(self, minio: dict, **kwargs): self.minio = minio # 将代码生成的文件同步到本地的路径 self.local_sync_path = kwargs.get("local_sync_path", None) + # Object-storage prefix of the session workspace (``workspace/``). + # Set by the linsight tool binder; empty for every other caller, which + # simply disables the mirror in ``sync_to_workspace``. + self.workspace_prefix = kwargs.get("workspace_prefix", None) @abstractmethod def run(self, code: str) -> Any: @@ -148,6 +153,69 @@ def relocation_advisory(moved: list[tuple[str, str]]) -> str: lines = "\n".join(f"- {old} -> {new}" for old, new in moved) return RELOCATED_PATH_NOTICE_HEADER + lines + def sync_to_workspace(self, dir_path: str, rel_paths: list[str]) -> int: + """Mirror this run's files into the session workspace prefix. Returns the count. + + The executor writes to a LOCAL working dir that is deleted when the task + ends (``_cleanup_resources``), while ``workspace//`` in object storage + is what the file tools (``ls`` / ``read_file``) actually see and what the + next turn inherits via ``seed_workspace_from_previous``. Without this + mirror a code-generated deliverable exists ONLY on that local disk, which + produced two long-standing defects: + + * the model cannot ``ls`` the file it just wrote (the prompt had to carry + an explicit "trust exitcode 0, do not go looking for it" caveat); + * a follow-up turn runs under a fresh svid whose workspace seeds from the + previous one's ``output/`` — empty, because nothing was ever written + there — so "把封面加个副标题" finds no deck and can only regenerate. + + Deletions are deliberately NOT mirrored: this runs off a created/modified + diff, and reconciling removals would mean trusting a partially-failed run + to delete objects the harvester may still need. + + Best-effort by design — a mirror failure must never fail the run. The local + copy is still what ``get_final_result_file`` harvests, so the user gets the + deliverable either way. + """ + if not self.minio or not self.workspace_prefix or not rel_paths: + return 0 + + bucket = self.minio.get("public_bucket") or "bisheng" + prefix = self.workspace_prefix.strip("/") + try: + minio_client = self._minio_client() + except Exception: + logger.exception("workspace mirror: minio client init failed; local copy is unaffected") + return 0 + + synced = 0 + for rel in rel_paths: + # Normalise once: a leading slash would make os.path.join return the + # absolute path and silently skip the file. + rel_key = rel.replace(os.sep, "/").lstrip("/") + local_path = os.path.join(dir_path, rel_key) + if not os.path.isfile(local_path): + continue + object_name = f"{prefix}/{rel_key}" + try: + minio_client.fput_object(bucket_name=bucket, object_name=object_name, file_path=local_path) + synced += 1 + except Exception: + # One unmirrored file degrades ls/continuity for that file only. + logger.exception("workspace mirror failed for {}", object_name) + if synced: + logger.debug("workspace mirror: {} file(s) -> {}/{}", synced, bucket, prefix) + return synced + + def _minio_client(self) -> Minio: + return Minio( + endpoint=self.minio.get("endpoint"), + access_key=self.minio.get("access_key"), + secret_key=self.minio.get("secret_key"), + secure=self.minio.get("schema") or self.minio.get("secure"), + cert_check=self.minio.get("cert_check"), + ) + def upload_minio( self, object_name: str, diff --git a/src/backend/bisheng_langchain/gpts/tools/code_interpreter/local_executor.py b/src/backend/bisheng_langchain/gpts/tools/code_interpreter/local_executor.py index 1067b337c7..f53551cbb8 100644 --- a/src/backend/bisheng_langchain/gpts/tools/code_interpreter/local_executor.py +++ b/src/backend/bisheng_langchain/gpts/tools/code_interpreter/local_executor.py @@ -363,6 +363,9 @@ def run_with_dir(self, code: str, dir_path: str, lang: str) -> (int, str, list): continue file_ext = os.path.splitext(rel)[-1] file_list.append(self.upload_minio(f"{uuid.uuid4().hex}.{file_ext}", file_name)) + # Mirror the same set into the session workspace so the file tools and the + # next turn can see what this run produced (see sync_to_workspace). + self.sync_to_workspace(dir_path, touched) # 同步执行结果文件到本地同步目录 if self.local_sync_path and os.path.exists(self.local_sync_path): files_info = list(os.scandir(dir_path)) diff --git a/src/backend/scripts/README.md b/src/backend/scripts/README.md index 8b099c8383..cad94a9dad 100644 --- a/src/backend/scripts/README.md +++ b/src/backend/scripts/README.md @@ -448,6 +448,29 @@ config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_media_no_asr_ config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_media_no_asr_transcript.py --file-id 456 --apply ``` +### `converge_knowledge_rebuild_failed_state.py` + +把重建失败留下的**容器级** `FAILED` 状态收敛回 `PUBLISHED`。 + +背景：重建 worker 过去只要有一个文件重建失败，就把整个知识库/知识空间标成 +`KnowledgeState.FAILED`。这个标记本是给"下次自动重建"用的，但失败文件已经不能自动 +重建、必须走重新解析，所以它只剩副作用：权限目标解析要求容器处于 `PUBLISHED` +(`knowledge_permission_service.py`)，于是一个坏文件让整个空间在权限层面"不存在"， +接口报 19003「资源类型或 ID 无效」，用户进不去。服务端已不再写这个状态 +(`rebuild_knowledge_worker.py`)；本脚本收敛存量。 + +只改容器状态，**不碰文件状态**——哪些文件失败仍记录在 `knowledge_file` 上，重新解析 +要靠它。`--scope space`（默认）只处理知识空间；`all` 再加普通知识库与个人知识库。 +QA 知识库不在范围内，其 `FAILED` 由 `worker/knowledge/qa.py` 另行写入、语义未变。 + +Usage (from `src/backend/`，默认 dry-run): + +```bash +config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_knowledge_rebuild_failed_state.py +config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_knowledge_rebuild_failed_state.py --apply +config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_knowledge_rebuild_failed_state.py --scope all --apply +``` + ### `backfill_knowledge_space_user_pin.py` F037：知识空间置顶从 `space_channel_member.is_pinned` 解耦到独立的 `knowledge_space_user_pin` 表（置顶是纯个人偏好，不再寄生在成员关系上）。本脚本把历史置顶迁移到新表，让升级后用户保留已置顶的空间。 diff --git a/src/backend/scripts/converge_knowledge_rebuild_failed_state.py b/src/backend/scripts/converge_knowledge_rebuild_failed_state.py new file mode 100644 index 0000000000..c25eeaaab7 --- /dev/null +++ b/src/backend/scripts/converge_knowledge_rebuild_failed_state.py @@ -0,0 +1,118 @@ +"""把重建失败留下的容器级 FAILED 状态收敛回 PUBLISHED。 + +重建 worker 过去在任何一个文件重建失败时，会把**整个**知识库/知识空间标成 +``KnowledgeState.FAILED``。这个标记原本是给"下次自动重建"用的，但失败文件现在已经 +不能自动重建、必须走重新解析，所以它不再有任何用途，只剩副作用：权限目标解析要求 +容器处于 ``PUBLISHED``（``knowledge_permission_service.py``），于是一个坏文件就让整 +个知识空间在权限层面"不存在"，接口报 19003「资源类型或 ID 无效」，用户根本进不去。 + +服务端已经不再写这个状态（``rebuild_knowledge_worker.py``）；本脚本收敛存量数据。 + +**只动容器状态，不碰文件状态**：哪些文件重建失败仍然记录在 ``knowledge_file`` 上， +那才是准确的信息，重新解析时要靠它。 + +Usage (from ``src/backend/``，默认 dry-run): + +```bash +config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_knowledge_rebuild_failed_state.py +config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_knowledge_rebuild_failed_state.py --apply +config=config.yaml PYTHONPATH=./ .venv/bin/python scripts/converge_knowledge_rebuild_failed_state.py --scope all --apply +``` + +``--scope space``（默认）只收敛知识空间；``all`` 还包括普通知识库和个人知识库 —— +它们由同一个 worker 标记，同样受影响。QA 知识库不在范围内：它的 FAILED 由 +``worker/knowledge/qa.py`` 另行写入，语义未变。 +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys + +_BACKEND_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if _BACKEND_ROOT not in sys.path: + sys.path.insert(0, _BACKEND_ROOT) + +from sqlalchemy import text # noqa: E402 + +from bisheng.core.context.tenant import bypass_tenant_filter # noqa: E402 +from bisheng.core.database import get_async_db_session # noqa: E402 +from bisheng.knowledge.domain.models.knowledge import ( # noqa: E402 + KnowledgeState, + KnowledgeTypeEnum, +) + +SCOPES: dict[str, tuple[KnowledgeTypeEnum, ...]] = { + "space": (KnowledgeTypeEnum.SPACE,), + "all": ( + KnowledgeTypeEnum.SPACE, + KnowledgeTypeEnum.NORMAL, + KnowledgeTypeEnum.PRIVATE, + ), +} + + +async def run(scope: str, apply: bool) -> int: + types = SCOPES[scope] + type_values = [item.value for item in types] + failed = KnowledgeState.FAILED.value + published = KnowledgeState.PUBLISHED.value + + with bypass_tenant_filter(): + async with get_async_db_session() as session: + rows = ( + await session.execute( + text( + "SELECT id, name, type, tenant_id FROM knowledge " + "WHERE state = :failed AND type IN :types ORDER BY type, id" + ).bindparams(failed=failed, types=tuple(type_values)) + ) + ).all() + + print(f"scope={scope} 命中 {len(rows)} 条容器级 FAILED 记录") + for row in rows: + print(f" id={row[0]:<8} type={KnowledgeTypeEnum(row[2]).name:<8} tenant={row[3]} {row[1]}") + + if not rows: + print("无需改动") + return 0 + if not apply: + print("\n[dry-run] 未写入任何数据；加 --apply 执行") + return 0 + + result = await session.execute( + text("UPDATE knowledge SET state = :published WHERE state = :failed AND type IN :types").bindparams( + published=published, failed=failed, types=tuple(type_values) + ) + ) + await session.commit() + print(f"\n已更新 {result.rowcount} 条 -> PUBLISHED") + + remaining = ( + await session.execute( + text("SELECT COUNT(*) FROM knowledge WHERE state = :failed AND type IN :types").bindparams( + failed=failed, types=tuple(type_values) + ) + ) + ).scalar_one() + print(f"剩余容器级 FAILED：{remaining}（应为 0）") + return 0 if remaining == 0 else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--scope", + choices=sorted(SCOPES), + default="space", + help="space=仅知识空间（默认）；all=再加普通知识库与个人知识库", + ) + parser.add_argument("--apply", action="store_true", help="执行写入；默认只报告") + args = parser.parse_args() + return asyncio.run(run(args.scope, args.apply)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/backend/test/knowledge/rag/test_excel_images.py b/src/backend/test/knowledge/rag/test_excel_images.py new file mode 100644 index 0000000000..a4116b94db --- /dev/null +++ b/src/backend/test/knowledge/rag/test_excel_images.py @@ -0,0 +1,183 @@ +"""Pictures embedded in a workbook must be staged for MinIO and attributed to +the sheet that actually owns them.""" + +import os +import zipfile + +import openpyxl +import pytest +from openpyxl.drawing.image import Image as OpenpyxlImage +from PIL import Image as PILImage + +from bisheng.knowledge.rag.pipeline.loader.base import BaseBishengLoader +from bisheng.knowledge.rag.pipeline.loader.excel import ExcelLoader +from bisheng.knowledge.rag.pipeline.loader.utils.excel_images import extract_excel_images + +PNG_BYTES = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06" + b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\xdac\xfc\x0f\x00\x01\x03\x01" + b"\x00\x18\xdd\x8d\xb0\x00\x00\x00\x00IEND\xaeB`\x82" +) + + +def _write_synthetic_workbook(path: str) -> None: + """A package where sheet order, r:id and file numbering all disagree. + + "Data" is sheet7.xml and owns nothing; "Cover" is sheet2.xml, has no cells + at all and owns the only picture. Pairing sheets with drawings positionally + would attribute the picture to the wrong sheet. + """ + with zipfile.ZipFile(path, "w") as zf: + zf.writestr( + "xl/workbook.xml", + '' + '' + '' + '', + ) + zf.writestr( + "xl/_rels/workbook.xml.rels", + '' + '' + '' + '', + ) + zf.writestr("xl/worksheets/sheet7.xml", '') + zf.writestr("xl/worksheets/sheet2.xml", '') + zf.writestr( + "xl/worksheets/_rels/sheet2.xml.rels", + '' + '' + '', + ) + # Namespace prefixes deliberately renamed: "xdr:"/"a:" are conventions, not rules. + zf.writestr( + "xl/drawings/drawing3.xml", + '' + '' + "37" + '' + "", + ) + zf.writestr( + "xl/drawings/_rels/drawing3.xml.rels", + '' + '' + '', + ) + zf.writestr("xl/media/image1.png", PNG_BYTES) + + +def _write_real_workbook(path: str, png_path: str) -> None: + """A workbook openpyxl itself produces: picture-only first sheet, data second.""" + PILImage.new("RGB", (4, 4), "white").save(png_path) + + workbook = openpyxl.Workbook() + cover = workbook.active + cover.title = "Portlet" + cover.add_image(OpenpyxlImage(png_path), "B2") + + data = workbook.create_sheet("Prices") + data.append(["date", "close"]) + data.append(["2026-05-08", 15.99]) + workbook.save(path) + + +def test_picture_is_attributed_to_the_sheet_that_owns_it(tmp_path): + xlsx = str(tmp_path / "synthetic.xlsx") + _write_synthetic_workbook(xlsx) + + images = extract_excel_images(xlsx) + + assert [(img.sheet_name, img.media_name) for img in images] == [("Cover", "image1.png")] + assert images[0].content == PNG_BYTES + + +def test_non_package_files_are_skipped_quietly(tmp_path): + csv_path = tmp_path / "plain.csv" + csv_path.write_text("a,b\n1,2\n", encoding="utf-8") + + assert extract_excel_images(str(csv_path)) == [] + assert extract_excel_images(str(tmp_path / "missing.xlsx")) == [] + + +def test_loader_stages_pictures_and_emits_a_chunk_per_sheet(tmp_path, monkeypatch): + monkeypatch.setattr(BaseBishengLoader, "_minio_bucket", property(lambda self: "bisheng")) + + xlsx = str(tmp_path / "report.xlsx") + _write_real_workbook(xlsx, str(tmp_path / "pic.png")) + + loader = ExcelLoader( + file_path=xlsx, + file_metadata={"source": "report.xlsx"}, + file_extension="xlsx", + tmp_dir=str(tmp_path / "work"), + image_object_dir="knowledge/images/90030/90286", + ) + documents = loader.load() + + image_docs = [doc for doc in documents if doc.page_content.startswith("## Portlet")] + assert len(image_docs) == 1, "the picture-only sheet must still produce a chunk" + assert "![image1.png](/bisheng/knowledge/images/90030/90286/image1.png)" in image_docs[0].page_content + + # Bytes are staged locally; ImageUploadTransformer performs the upload. + assert os.listdir(loader.local_image_dir) == ["image1.png"] + + # The table sheet is still parsed, and chunk_index stays contiguous. + assert any("2026-05-08" in doc.page_content for doc in documents) + assert [doc.metadata["chunk_index"] for doc in documents] == list(range(len(documents))) + + +def test_loader_leaves_pictureless_workbooks_untouched(tmp_path, monkeypatch): + monkeypatch.setattr(BaseBishengLoader, "_minio_bucket", property(lambda self: "bisheng")) + + xlsx = str(tmp_path / "plain.xlsx") + workbook = openpyxl.Workbook() + workbook.active.append(["date", "close"]) + workbook.active.append(["2026-05-08", 15.99]) + workbook.save(xlsx) + + loader = ExcelLoader( + file_path=xlsx, + file_metadata={}, + file_extension="xlsx", + tmp_dir=str(tmp_path / "work"), + image_object_dir="knowledge/images/1/2", + ) + documents = loader.load() + + assert documents, "table content must still be produced" + assert all(not doc.page_content.startswith("## ") for doc in documents) + # No staging dir is created when there is nothing to stage. + assert loader.local_image_dir is None + + +@pytest.mark.parametrize("extension", ["csv"]) +def test_image_extraction_is_limited_to_xlsx(tmp_path, extension, monkeypatch): + """csv/xls never carry an OPC drawing part; the loader must not even look.""" + calls = [] + monkeypatch.setattr( + "bisheng.knowledge.rag.pipeline.loader.excel.extract_excel_images", + lambda path: calls.append(path) or [], + ) + + csv_path = tmp_path / f"data.{extension}" + csv_path.write_text("date,close\n2026-05-08,15.99\n", encoding="utf-8") + + loader = ExcelLoader( + file_path=str(csv_path), + file_metadata={}, + file_extension=extension, + tmp_dir=str(tmp_path / "work"), + image_object_dir="knowledge/images/1/2", + ) + loader.load() + + assert calls == [] diff --git a/src/backend/test/knowledge/test_permission_target_rejection_reason.py b/src/backend/test/knowledge/test_permission_target_rejection_reason.py new file mode 100644 index 0000000000..d70aa25520 --- /dev/null +++ b/src/backend/test/knowledge/test_permission_target_rejection_reason.py @@ -0,0 +1,99 @@ +"""Rejecting a permission target must say why — in the log, not the response. + +19003 covers seven different causes with one sentence, "Invalid resource type or +ID". A knowledge space stranded at status=FAILED therefore read exactly like a +mistyped id, and finding the real cause meant querying the database by hand. + +The response stays uniform on purpose: `permission_error_response` flattens every +rejection so a missing resource and one in another tenant remain +indistinguishable. The reason goes to the server log instead. +""" + +from __future__ import annotations + +import logging + +import pytest + +from bisheng.common.errcode.permission import PermissionInvalidResourceError +from bisheng.knowledge.domain.services import knowledge_permission_service as module +from bisheng.knowledge.domain.services.knowledge_permission_service import ( + KnowledgeContainerPermissionRecord, +) +from bisheng.permission.domain.services.permission_action_service import PermissionActor + +ACTOR = PermissionActor(user_id=7, current_tenant_id=1) + + +def _record(**overrides) -> KnowledgeContainerPermissionRecord: + fields = { + "tenant_id": 1, + "resource_type": "knowledge_space", + "resource_id": "4149", + "status": "PUBLISHED", + "kind": "SPACE", + "owner_user_id": 3, + "permission_version": 0, + "context_version": "v1", + } + fields.update(overrides) + return KnowledgeContainerPermissionRecord(**fields) + + +@pytest.mark.parametrize( + ("record", "expected"), + [ + (None, "NOT_FOUND"), + (_record(kind="NORMAL"), "KIND_MISMATCH:NORMAL"), + (_record(status="FAILED"), "STATUS_NOT_USABLE:FAILED"), + (_record(status="REBUILDING"), "STATUS_NOT_USABLE:REBUILDING"), + (_record(tenant_id=99), "TENANT_MISMATCH"), + (_record(resource_id="9999"), "IDENTITY_MISMATCH"), + ], +) +def test_each_cause_is_named(record, expected) -> None: + reason = module._container_rejection( + record, + ACTOR, + "knowledge_space", + "4149", + allowed_statuses={"PUBLISHED"}, + ) + assert reason == expected + + +def test_a_usable_container_is_not_rejected() -> None: + assert ( + module._container_rejection( + _record(), + ACTOR, + "knowledge_space", + "4149", + allowed_statuses={"PUBLISHED"}, + ) + is None + ) + + +def test_a_super_admin_crosses_tenants() -> None: + assert ( + module._container_rejection( + _record(tenant_id=99), + PermissionActor(user_id=1, current_tenant_id=1, super_admin=True), + "knowledge_space", + "4149", + allowed_statuses={"PUBLISHED"}, + ) + is None + ) + + +def test_the_reason_is_logged_and_kept_out_of_the_error(caplog) -> None: + with caplog.at_level(logging.WARNING): + error = module._reject_container("STATUS_NOT_USABLE:FAILED", "knowledge_space", "4149") + + assert isinstance(error, PermissionInvalidResourceError) + assert "STATUS_NOT_USABLE:FAILED" in caplog.text + assert "knowledge_space:4149" in caplog.text + # The caller still sees the single opaque business message. + assert "STATUS_NOT_USABLE" not in error.Msg diff --git a/src/backend/test/knowledge/test_rebuild_keeps_container_usable.py b/src/backend/test/knowledge/test_rebuild_keeps_container_usable.py new file mode 100644 index 0000000000..3f28899c18 --- /dev/null +++ b/src/backend/test/knowledge/test_rebuild_keeps_container_usable.py @@ -0,0 +1,84 @@ +"""A failed file must not take its knowledge container down with it. + +The rebuild worker used to mark the whole knowledge FAILED as soon as one file +failed. That flag existed to drive a later automatic rebuild, which no longer +happens — a failed file is recovered by re-parsing it. Meanwhile permission +target resolution requires the container to be PUBLISHED, so the flag made a +whole knowledge space unreachable and reported it as "invalid resource type or +ID". Twenty-one spaces were stranded that way. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from bisheng.knowledge.domain.models.knowledge import KnowledgeState + +_WORKER = "bisheng.worker.knowledge.rebuild_knowledge_worker" + + +@pytest.fixture +def knowledge(): + return SimpleNamespace(id=4149, state=KnowledgeState.REBUILDING.value, model=7) + + +def _run(knowledge, *, success, failed, raise_on_rebuild=None): + files = [SimpleNamespace(id=file_id, status=0, remark=None) for file_id in (*success, *failed)] + saved: list[object] = [] + + def rebuild(*args, **kwargs): + del args, kwargs + if raise_on_rebuild is not None: + raise raise_on_rebuild + return list(success), list(failed) + + with ( + patch(f"{_WORKER}.KnowledgeDao") as dao, + patch(f"{_WORKER}.KnowledgeFileDao") as file_dao, + patch(f"{_WORKER}._rebuild_embeddings", side_effect=rebuild), + patch(f"{_WORKER}._delete_es_files"), + patch(f"{_WORKER}.KnowledgeService"), + ): + dao.query_by_id.return_value = knowledge + dao.update_one.side_effect = lambda row: saved.append(row) + file_dao.get_files_by_multiple_status.return_value = files + + from bisheng.worker.knowledge import rebuild_knowledge_worker + + try: + rebuild_knowledge_worker.rebuild_knowledge_celery.run(knowledge.id, 9, 1) + except Exception as exc: # surfaced to Celery; state must still be sane + return files, exc + return files, None + + +def test_a_partly_failed_rebuild_leaves_the_container_published(knowledge) -> None: + files, error = _run(knowledge, success=[1, 2], failed=[3]) + + assert error is None + assert knowledge.state == KnowledgeState.PUBLISHED.value + # The failure is recorded where it actually happened. + failed_file = next(item for item in files if item.id == 3) + assert failed_file.status == 3 + + +def test_an_unexpected_error_does_not_strand_the_container(knowledge) -> None: + # Needs files, otherwise the worker returns before it ever rebuilds. + files, error = _run(knowledge, success=[1, 2], failed=[], raise_on_rebuild=RuntimeError("milvus down")) + + del files + assert isinstance(error, RuntimeError) + # Neither FAILED nor stuck in REBUILDING. + assert knowledge.state == KnowledgeState.PUBLISHED.value + + +def test_the_worker_no_longer_writes_a_container_failed_state() -> None: + """Guard the intent itself, not just the two paths exercised above.""" + + from pathlib import Path + + source = Path(f"bisheng/worker/knowledge/{'rebuild_knowledge_worker'}.py").read_text(encoding="utf-8") + assert "KnowledgeState.FAILED" not in source diff --git a/src/backend/test/linsight/test_builtin_skill_scripts.py b/src/backend/test/linsight/test_builtin_skill_scripts.py new file mode 100644 index 0000000000..2a1463afa7 --- /dev/null +++ b/src/backend/test/linsight/test_builtin_skill_scripts.py @@ -0,0 +1,615 @@ +"""Regression guard for the three built-in skill bundles' checker scripts. + +These scripts are the only mechanical quality gate the Linsight model has when it +builds a .pptx / .xlsx / .docx, and they were shipped with zero tests: a threshold +tweak, a regex edit, or an accidental non-zero exit could silently break them and +nothing in CI would notice. + +Three properties are locked down here, each for a failure mode that has actually +happened in this bundle family: + +* **Bidirectional.** A bad sample must be *rejected* (``结论: 不通过`` plus the + specific rule's wording) AND a good sample must be *accepted* (``结论: 通过``). + Testing only the bad direction lets a checker degrade into "always shouts", + which is worse than no checker — the model learns to ignore the report. +* **``returncode == 0``, always**, including on a corrupt file. The BiSheng code + executor is either/or: on a non-zero exit it returns *stderr* and throws the + whole stdout report away. A script that "correctly" exits 1 on findings makes + its own report invisible. +* **No dependency on LibreOffice.** The recalc/render scripts are exercised with + ``soffice`` stripped from ``PATH``, asserting the documented Chinese fallback + text rather than a crash. CI must never need LibreOffice installed. + +Samples are built with python-pptx / openpyxl / python-docx, which are backend +dependencies, and the .docx good sample is built with the bundle's own +``docx_helpers`` — "the helper's own output passes the bundle's own checker" is +the contract that matters. +""" + +from __future__ import annotations + +import importlib.util +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest +from docx import Document +from docx.shared import Cm as DocxCm +from docx.shared import Pt as DocxPt +from openpyxl import Workbook +from openpyxl.styles import Font +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE +from pptx.enum.text import MSO_AUTO_SIZE +from pptx.util import Inches, Pt + +from bisheng.linsight.domain.services.builtin_skill_seeder import BUILTIN_SKILLS_DIR + +PASS = "结论: 通过" +FAIL = "结论: 不通过" + + +# --------------------------------------------------------------------------- # +# running a packaged script the way SKILL.md tells the model to run it +# --------------------------------------------------------------------------- # +def run_script(bundle: str, script: str, *args, cwd: Path | None = None, path_env: str | None = None): + """Run ``skills//scripts/