修复订阅词库的列解析和词频丢失

CHANGELOG.en.md

@@ -12,6 +12,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
 
 ## [Unreleased]
 
+### Fixed
+
+- Subscribed dictionaries with a non-default column order could have words, readings or weights read incorrectly. AIME now follows the declared column order, accepts files with a leading BOM and undeclared two-column word/weight tables, and keeps the highest weight for duplicate entries. Existing subscriptions are rebuilt on the next deployment without downloading them again.
+
 ## [0.1.9] - 2026-10-13
 
 > Stable release · Developer ID signed · notarized by Apple

CHANGELOG.md

@@ -11,6 +11,10 @@ AIME 每个版本面向用户的变动都记在这里，官网「更新日志」
 
 ## [Unreleased]
 
+### 修复
+
+- 订阅词库的列顺序与默认格式不同时，词条、读音和词频可能读错。现在按词库声明的列顺序读取，兼容带 BOM 的文件和无列声明的两列词频表，重复词条保留较高词频；已有订阅会在下次部署时重新生成词表，无需重新下载。
+
 ## [0.1.9] - 2026-10-13
 
 > 正式版 · Developer ID 签名 · 已通过 Apple 公证

Packages/AIMEKit/Sources/AIMECore/VocabularySubscriptions.swift

@@ -16,31 +16,79 @@ public struct VocabularyEntry: Sendable, Hashable {
 
 public enum VocabularyParser {
     /// Parses a Rime `dict.yaml`, a `text<TAB>code[<TAB>weight]` table, or a plain word
-    /// list (pinyin computed locally). Comment and header lines are skipped.
+    /// list (pinyin computed locally). Declared dictionary columns override the TSV order;
+    /// malformed headers return no entries rather than guessing column roles.
     public static func parse(_ content: String) -> [VocabularyEntry] {
-        var lines = content.components(separatedBy: .newlines)
-        if let end = lines.firstIndex(where: { $0.trimmingCharacters(in: .whitespaces) == "..." }),
-           lines.prefix(end).contains(where: { $0.hasPrefix("name:") || $0 == "---" }) {
-            lines = Array(lines[(end + 1)...])
-        }
-        var seen = Set<String>()
+        let content = content.hasPrefix("\u{feff}") ? String(content.dropFirst()) : content
+        guard let (lines, declaredColumns) = splitHeader(content.components(separatedBy: .newlines)) else { return [] }
+        let textIndex = declaredColumns?.firstIndex(of: "text") ?? 0
+        let codeIndex = declaredColumns == nil ? 1 : declaredColumns?.firstIndex(of: "code")
+        let weightIndex = declaredColumns?.firstIndex(of: "weight")
+        var seen: [String: Int] = [:]
         var entries: [VocabularyEntry] = []
         for raw in lines {
             let line = raw.trimmingCharacters(in: .whitespaces)
             if line.isEmpty || line.hasPrefix("#") { continue }
-            let columns = line.components(separatedBy: "\t")
-            let text = columns[0].trimmingCharacters(in: .whitespaces)
+            // Split before trimming so empty fields do not shift the following columns.
+            let columns = raw.components(separatedBy: "\t").map { $0.trimmingCharacters(in: .whitespaces) }
+            func field(_ index: Int?) -> String {
+                guard let index, columns.indices.contains(index) else { return "" }
+                return columns[index]
+            }
+            let text = field(textIndex)
             guard !text.isEmpty, text.count <= 32 else { continue }
-            var code = columns.count > 1 ? columns[1].trimmingCharacters(in: .whitespaces) : ""
-            // `text code weight`, or `text code tag weight` (aime_tech): first numeric column.
-            let weight = columns.dropFirst(2).lazy.compactMap { Int($0.trimmingCharacters(in: .whitespaces)) }.first ?? 50
-            if code.isEmpty || Int(code) != nil { code = pinyin(for: text) ?? "" }
-            guard !code.isEmpty, seen.insert("\(text)\t\(code)").inserted else { continue }
-            entries.append(VocabularyEntry(text: text, pinyin: code.lowercased(), weight: weight))
+            var code = field(codeIndex)
+            // Undeclared tables accept `text weight`, `text code weight` and `text code tag weight` (aime_tech).
+            let weight = if declaredColumns != nil {
+                Int(field(weightIndex)) ?? 50
+            } else if columns.count == 2 {
+                Int(code) ?? 50
+            } else {
+                columns.dropFirst(2).lazy.compactMap { Int($0) }.first ?? 50
+            }
+            if code.isEmpty || (declaredColumns == nil && Int(code) != nil) { code = pinyin(for: text) ?? "" }
+            code = code.split(whereSeparator: \.isWhitespace).joined(separator: " ").lowercased()
+            guard !code.isEmpty else { continue }
+            let key = "\(text)\t\(code)"
+            if let index = seen[key] {
+                entries[index].weight = max(entries[index].weight, weight)
+            } else {
+                seen[key] = entries.count
+                entries.append(VocabularyEntry(text: text, pinyin: code, weight: weight))
+            }
         }
         return entries
     }
 
+    private static func splitHeader(_ lines: [String]) -> (ArraySlice<String>, [String]?)? {
+        guard let end = lines.firstIndex(where: { $0.trimmingCharacters(in: .whitespaces) == "..." }),
+              lines.prefix(end).contains(where: { $0.hasPrefix("name:") || $0 == "---" }) else {
+            let first = lines.lazy.map { $0.trimmingCharacters(in: .whitespaces) }
+                .first { !$0.isEmpty && !$0.hasPrefix("#") }
+            // A truncated header is not a list of words to auto-annotate.
+            if first == "---" || first?.hasPrefix("name:") == true { return nil }
+            return (lines[...], nil)
+        }
+        let headerLines = lines.prefix(end)
+        // Parse only the small YAML header, never the vocabulary body.
+        guard headerLines.reduce(0, { $0 + $1.utf8.count + 1 }) <= 64 * 1024,
+              let header = try? ConfigValue.parse(yaml: headerLines.joined(separator: "\n")),
+              let fields = header.entries,
+              fields.filter({ $0.key == "columns" }).count <= 1 else { return nil }
+        var columns: [String]?
+        if let value = header["columns"] {
+            guard let values = value.listValue else { return nil }
+            var names: [String] = []
+            for value in values {
+                guard case let .string(name) = value, !name.isEmpty else { return nil }
+                names.append(name)
+            }
+            guard names.contains("text"), Set(names).count == names.count else { return nil }
+            columns = names
+        }
+        return (lines[(end + 1)...], columns)
+    }
+
     /// Space-separated toneless pinyin for Chinese text; lowercase alphanumerics for Latin.
     public static func pinyin(for text: String) -> String? {
         let hasHan = text.unicodeScalars.contains { (0x4E00...0x9FFF).contains($0.value) }
@@ -563,7 +611,7 @@ public struct SubscriptionManager: Sendable {
 
     func tableSignature(schemas: [String], items: [VocabularySubscription]) -> String {
         func digest(_ url: URL) -> String { (try? PackageManager.sha256(of: url)) ?? "missing" }
-        var parts = ["v2", schemas.sorted().joined(separator: ",")]
+        var parts = ["v3", schemas.sorted().joined(separator: ",")]
         if let shipped = paths.sharedDataDir?.appendingPathComponent("aime/aime_tech.tsv") { parts.append(digest(shipped)) }
         for item in items.sorted(by: { $0.id < $1.id }) { parts.append("\(item.id)=\(digest(cacheURL(item.id)))") }
         return parts.joined(separator: "|")

Packages/AIMEKit/Tests/AIMECoreTests/VocabularyParserTests.swift

@@ -0,0 +1,114 @@
+@testable import AIMECore
+import Foundation
+import Testing
+
+struct VocabularyParserTests {
+    @Test func keepsHighestDuplicateWeightAndFirstOccurrenceOrder() {
+        let parsed = VocabularyParser.parse("新词\txin ci\t20\n旧词\tjiu ci\t60\n新词\txin ci\t90\n新词\txin ci\t10\n")
+        #expect(parsed == [
+            VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 90),
+            VocabularyEntry(text: "旧词", pinyin: "jiu ci", weight: 60)
+        ])
+    }
+
+    @Test func normalizesCodesWithoutMergingDifferentReadings() {
+        let parsed = VocabularyParser.parse("银行\t YIN   HANG \t20\n银行\tyin\u{00a0}hang\t90\n银行\tyin xing\t30\n")
+        #expect(parsed == [
+            VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90),
+            VocabularyEntry(text: "银行", pinyin: "yin xing", weight: 30)
+        ])
+    }
+
+    @Test(arguments: ["[weight, text, code, stem]", "\n  - weight\n  - text\n  - code\n  - stem"])
+    func honorsDeclaredColumnOrder(columns: String) {
+        let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: \(columns)\n...\n90\t银行\tyin hang\tignored\n")
+        #expect(parsed == [VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90)])
+    }
+
+    @Test func preservesDeclaredNumericCodes() {
+        let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: [text, code, weight]\n...\n数字\t123\t80\n")
+        #expect(parsed == [VocabularyEntry(text: "数字", pinyin: "123", weight: 80)])
+        #expect(VocabularyParser.parse("---\nname: demo\ncolumns: [text, code]\n...\n数字\t123\n")
+            == [VocabularyEntry(text: "数字", pinyin: "123")])
+    }
+
+    @Test func preservesWeightsInUndeclaredTwoColumnTables() {
+        #expect(VocabularyParser.parse("新词\t 100 \n银行\tyin hang\n") == [
+            VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 100),
+            VocabularyEntry(text: "银行", pinyin: "yin hang")
+        ])
+    }
+
+    @Test func keepsEmptyFieldsInTheirDeclaredColumns() {
+        let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: [code, text, weight]\n...\n\t新词\t90\nxin ci\t\t70\n")
+        #expect(parsed == [VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 90)])
+        #expect(VocabularyParser.parse("\txin ci\t90\n").isEmpty)
+    }
+
+    @Test func fillsMissingCodeWithoutGuessingWeightFromOtherColumns() {
+        let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: [text, stem, weight]\n...\n新词\t123\t90\n旧词\t456\n")
+        #expect(parsed == [
+            VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 90),
+            VocabularyEntry(text: "旧词", pinyin: "jiu ci")
+        ])
+    }
+
+    @Test func preservesLegacyTablesAndPlainLists() {
+        #expect(VocabularyParser.parse("智能体\tzhi neng ti\tai_term\t95\n")
+            == [VocabularyEntry(text: "智能体", pinyin: "zhi neng ti", weight: 95)])
+        #expect(VocabularyParser.parse("---\nname: demo\n...\n银行\tyin hang\t90\n")
+            == [VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90)])
+        #expect(VocabularyParser.parse("# comment\n\n新词\nClaude\n") == [
+            VocabularyEntry(text: "新词", pinyin: "xin ci"),
+            VocabularyEntry(text: "Claude", pinyin: "claude")
+        ])
+    }
+
+    @Test func stripsLeadingByteOrderMark() {
+        #expect(VocabularyParser.parse("\u{feff}新词\n") == [VocabularyEntry(text: "新词", pinyin: "xin ci")])
+        let parsed = VocabularyParser.parse("\u{feff}# comment\n---\nname: demo\ncolumns: [text, weight, code]\n...\n银行\t90\tyin hang\n")
+        #expect(parsed == [VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90)])
+    }
+
+    @Test(arguments: ["[text, code, code]", "[weight, code]", "[text, 1]", "[]", "text", "[text, code"])
+    func rejectsInvalidColumns(columns: String) {
+        #expect(VocabularyParser.parse("---\nname: demo\ncolumns: \(columns)\n...\n银行\tyin hang\t90\n").isEmpty)
+    }
+
+    @Test func rejectsAmbiguousOrIncompleteHeaders() {
+        let duplicate = "---\nname: demo\ncolumns: [text, code]\ncolumns: [code, text]\n...\n银行\tyin hang\n"
+        #expect(VocabularyParser.parse(duplicate).isEmpty)
+        #expect(VocabularyParser.parse("# comment\n---\nname: demo\ncolumns: [text, code]\n银行\tyin hang\n").isEmpty)
+        let oversized = "---\nname: demo\ndescription: " + String(repeating: "x", count: 64 * 1024) + "\n...\n银行\tyin hang\n"
+        #expect(VocabularyParser.parse(oversized).isEmpty)
+    }
+
+    @Test func rebuildsExistingTablesWithTheCorrectedParser() throws {
+        let fm = FileManager.default
+        let root = fm.temporaryDirectory.appendingPathComponent("aime-parser-\(UUID().uuidString)")
+        defer { try? fm.removeItem(at: root) }
+        let paths = AIMEPaths(userDataDir: root)
+        let manager = SubscriptionManager(paths: paths)
+        let item = VocabularySubscription(id: "sample", name: "Sample", url: URL(string: "https://example.invalid/dict.yaml")!, addedAt: Date())
+        try manager.save([item])
+        let cache = manager.cacheURL(item.id)
+        try fm.createDirectory(at: cache.deletingLastPathComponent(), withIntermediateDirectories: true)
+        let content = Data("\u{feff}---\nname: demo\ncolumns: [text, weight, code]\n...\n银行\t20\tyin hang\n银行\t90\tyin hang\n".utf8)
+        try content.write(to: cache)
+        let modified = try cache.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate
+        try "schema_list:\n  - schema: rime_ice\n".write(to: root.appendingPathComponent("default.yaml"), atomically: true, encoding: .utf8)
+        try manager.rebuild(schemas: ["rime_ice"])
+        let table = root.appendingPathComponent("aime_online.txt")
+        try "银行\tyinxing\t50\n".write(to: table, atomically: true, encoding: .utf8)
+        let signature = manager.tableSignature(schemas: ["rime_ice"], items: [item])
+        let oldSignature = "v2|" + signature.split(separator: "|", maxSplits: 1)[1]
+        try oldSignature.write(to: paths.aimeDir.appendingPathComponent("subscriptions/.signature"), atomically: true, encoding: .utf8)
+
+        #expect(try manager.ensureTables())
+        let rows = try String(contentsOf: table, encoding: .utf8).split(separator: "\n").filter { !$0.hasPrefix("#") }
+        #expect(rows == ["银行\tyinhang\t90"])
+        #expect(try Data(contentsOf: cache) == content)
+        #expect(try cache.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate == modified)
+        #expect(try manager.ensureTables() == false)
+    }
+}