diff --git a/CHANGELOG.en.md b/CHANGELOG.en.md index 41f4cbd..17ba76e 100644 --- a/CHANGELOG.en.md +++ b/CHANGELOG.en.md @@ -12,6 +12,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and ## [Unreleased] +### Fixed + +- Subscribed dictionaries with a non-default column order could have words, readings or weights read incorrectly. AIME now follows the declared column order, accepts files with a leading BOM and undeclared two-column word/weight tables, and keeps the highest weight for duplicate entries. Existing subscriptions are rebuilt on the next deployment without downloading them again. + ## [0.1.9] - 2026-10-13 > Stable release · Developer ID signed · notarized by Apple diff --git a/CHANGELOG.md b/CHANGELOG.md index 5429a0b..08d0042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ AIME 每个版本面向用户的变动都记在这里，官网「更新日志」 ## [Unreleased] +### 修复 + +- 订阅词库的列顺序与默认格式不同时，词条、读音和词频可能读错。现在按词库声明的列顺序读取，兼容带 BOM 的文件和无列声明的两列词频表，重复词条保留较高词频；已有订阅会在下次部署时重新生成词表，无需重新下载。 + ## [0.1.9] - 2026-10-13 > 正式版 · Developer ID 签名 · 已通过 Apple 公证 diff --git a/Packages/AIMEKit/Sources/AIMECore/VocabularySubscriptions.swift b/Packages/AIMEKit/Sources/AIMECore/VocabularySubscriptions.swift index 24c71d6..2c9b647 100644 --- a/Packages/AIMEKit/Sources/AIMECore/VocabularySubscriptions.swift +++ b/Packages/AIMEKit/Sources/AIMECore/VocabularySubscriptions.swift @@ -16,31 +16,79 @@ public struct VocabularyEntry: Sendable, Hashable { public enum VocabularyParser { /// Parses a Rime `dict.yaml`, a `textcode[weight]` table, or a plain word - /// list (pinyin computed locally). Comment and header lines are skipped. + /// list (pinyin computed locally). Declared dictionary columns override the TSV order; + /// malformed headers return no entries rather than guessing column roles. public static func parse(_ content: String) -> [VocabularyEntry] { - var lines = content.components(separatedBy: .newlines) - if let end = lines.firstIndex(where: { $0.trimmingCharacters(in: .whitespaces) == "..." }), - lines.prefix(end).contains(where: { $0.hasPrefix("name:") || $0 == "---" }) { - lines = Array(lines[(end + 1)...]) - } - var seen = Set() + let content = content.hasPrefix("\u{feff}") ? String(content.dropFirst()) : content + guard let (lines, declaredColumns) = splitHeader(content.components(separatedBy: .newlines)) else { return [] } + let textIndex = declaredColumns?.firstIndex(of: "text") ?? 0 + let codeIndex = declaredColumns == nil ? 1 : declaredColumns?.firstIndex(of: "code") + let weightIndex = declaredColumns?.firstIndex(of: "weight") + var seen: [String: Int] = [:] var entries: [VocabularyEntry] = [] for raw in lines { let line = raw.trimmingCharacters(in: .whitespaces) if line.isEmpty || line.hasPrefix("#") { continue } - let columns = line.components(separatedBy: "\t") - let text = columns[0].trimmingCharacters(in: .whitespaces) + // Split before trimming so empty fields do not shift the following columns. + let columns = raw.components(separatedBy: "\t").map { $0.trimmingCharacters(in: .whitespaces) } + func field(_ index: Int?) -> String { + guard let index, columns.indices.contains(index) else { return "" } + return columns[index] + } + let text = field(textIndex) guard !text.isEmpty, text.count <= 32 else { continue } - var code = columns.count > 1 ? columns[1].trimmingCharacters(in: .whitespaces) : "" - // `text code weight`, or `text code tag weight` (aime_tech): first numeric column. - let weight = columns.dropFirst(2).lazy.compactMap { Int($0.trimmingCharacters(in: .whitespaces)) }.first ?? 50 - if code.isEmpty || Int(code) != nil { code = pinyin(for: text) ?? "" } - guard !code.isEmpty, seen.insert("\(text)\t\(code)").inserted else { continue } - entries.append(VocabularyEntry(text: text, pinyin: code.lowercased(), weight: weight)) + var code = field(codeIndex) + // Undeclared tables accept `text weight`, `text code weight` and `text code tag weight` (aime_tech). + let weight = if declaredColumns != nil { + Int(field(weightIndex)) ?? 50 + } else if columns.count == 2 { + Int(code) ?? 50 + } else { + columns.dropFirst(2).lazy.compactMap { Int($0) }.first ?? 50 + } + if code.isEmpty || (declaredColumns == nil && Int(code) != nil) { code = pinyin(for: text) ?? "" } + code = code.split(whereSeparator: \.isWhitespace).joined(separator: " ").lowercased() + guard !code.isEmpty else { continue } + let key = "\(text)\t\(code)" + if let index = seen[key] { + entries[index].weight = max(entries[index].weight, weight) + } else { + seen[key] = entries.count + entries.append(VocabularyEntry(text: text, pinyin: code, weight: weight)) + } } return entries } + private static func splitHeader(_ lines: [String]) -> (ArraySlice, [String]?)? { + guard let end = lines.firstIndex(where: { $0.trimmingCharacters(in: .whitespaces) == "..." }), + lines.prefix(end).contains(where: { $0.hasPrefix("name:") || $0 == "---" }) else { + let first = lines.lazy.map { $0.trimmingCharacters(in: .whitespaces) } + .first { !$0.isEmpty && !$0.hasPrefix("#") } + // A truncated header is not a list of words to auto-annotate. + if first == "---" || first?.hasPrefix("name:") == true { return nil } + return (lines[...], nil) + } + let headerLines = lines.prefix(end) + // Parse only the small YAML header, never the vocabulary body. + guard headerLines.reduce(0, { $0 + $1.utf8.count + 1 }) <= 64 * 1024, + let header = try? ConfigValue.parse(yaml: headerLines.joined(separator: "\n")), + let fields = header.entries, + fields.filter({ $0.key == "columns" }).count <= 1 else { return nil } + var columns: [String]? + if let value = header["columns"] { + guard let values = value.listValue else { return nil } + var names: [String] = [] + for value in values { + guard case let .string(name) = value, !name.isEmpty else { return nil } + names.append(name) + } + guard names.contains("text"), Set(names).count == names.count else { return nil } + columns = names + } + return (lines[(end + 1)...], columns) + } + /// Space-separated toneless pinyin for Chinese text; lowercase alphanumerics for Latin. public static func pinyin(for text: String) -> String? { let hasHan = text.unicodeScalars.contains { (0x4E00...0x9FFF).contains($0.value) } @@ -563,7 +611,7 @@ public struct SubscriptionManager: Sendable { func tableSignature(schemas: [String], items: [VocabularySubscription]) -> String { func digest(_ url: URL) -> String { (try? PackageManager.sha256(of: url)) ?? "missing" } - var parts = ["v2", schemas.sorted().joined(separator: ",")] + var parts = ["v3", schemas.sorted().joined(separator: ",")] if let shipped = paths.sharedDataDir?.appendingPathComponent("aime/aime_tech.tsv") { parts.append(digest(shipped)) } for item in items.sorted(by: { $0.id < $1.id }) { parts.append("\(item.id)=\(digest(cacheURL(item.id)))") } return parts.joined(separator: "|") diff --git a/Packages/AIMEKit/Tests/AIMECoreTests/VocabularyParserTests.swift b/Packages/AIMEKit/Tests/AIMECoreTests/VocabularyParserTests.swift new file mode 100644 index 0000000..f2eea73 --- /dev/null +++ b/Packages/AIMEKit/Tests/AIMECoreTests/VocabularyParserTests.swift @@ -0,0 +1,114 @@ +@testable import AIMECore +import Foundation +import Testing + +struct VocabularyParserTests { + @Test func keepsHighestDuplicateWeightAndFirstOccurrenceOrder() { + let parsed = VocabularyParser.parse("新词\txin ci\t20\n旧词\tjiu ci\t60\n新词\txin ci\t90\n新词\txin ci\t10\n") + #expect(parsed == [ + VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 90), + VocabularyEntry(text: "旧词", pinyin: "jiu ci", weight: 60) + ]) + } + + @Test func normalizesCodesWithoutMergingDifferentReadings() { + let parsed = VocabularyParser.parse("银行\t YIN HANG \t20\n银行\tyin\u{00a0}hang\t90\n银行\tyin xing\t30\n") + #expect(parsed == [ + VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90), + VocabularyEntry(text: "银行", pinyin: "yin xing", weight: 30) + ]) + } + + @Test(arguments: ["[weight, text, code, stem]", "\n - weight\n - text\n - code\n - stem"]) + func honorsDeclaredColumnOrder(columns: String) { + let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: \(columns)\n...\n90\t银行\tyin hang\tignored\n") + #expect(parsed == [VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90)]) + } + + @Test func preservesDeclaredNumericCodes() { + let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: [text, code, weight]\n...\n数字\t123\t80\n") + #expect(parsed == [VocabularyEntry(text: "数字", pinyin: "123", weight: 80)]) + #expect(VocabularyParser.parse("---\nname: demo\ncolumns: [text, code]\n...\n数字\t123\n") + == [VocabularyEntry(text: "数字", pinyin: "123")]) + } + + @Test func preservesWeightsInUndeclaredTwoColumnTables() { + #expect(VocabularyParser.parse("新词\t 100 \n银行\tyin hang\n") == [ + VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 100), + VocabularyEntry(text: "银行", pinyin: "yin hang") + ]) + } + + @Test func keepsEmptyFieldsInTheirDeclaredColumns() { + let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: [code, text, weight]\n...\n\t新词\t90\nxin ci\t\t70\n") + #expect(parsed == [VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 90)]) + #expect(VocabularyParser.parse("\txin ci\t90\n").isEmpty) + } + + @Test func fillsMissingCodeWithoutGuessingWeightFromOtherColumns() { + let parsed = VocabularyParser.parse("---\nname: demo\ncolumns: [text, stem, weight]\n...\n新词\t123\t90\n旧词\t456\n") + #expect(parsed == [ + VocabularyEntry(text: "新词", pinyin: "xin ci", weight: 90), + VocabularyEntry(text: "旧词", pinyin: "jiu ci") + ]) + } + + @Test func preservesLegacyTablesAndPlainLists() { + #expect(VocabularyParser.parse("智能体\tzhi neng ti\tai_term\t95\n") + == [VocabularyEntry(text: "智能体", pinyin: "zhi neng ti", weight: 95)]) + #expect(VocabularyParser.parse("---\nname: demo\n...\n银行\tyin hang\t90\n") + == [VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90)]) + #expect(VocabularyParser.parse("# comment\n\n新词\nClaude\n") == [ + VocabularyEntry(text: "新词", pinyin: "xin ci"), + VocabularyEntry(text: "Claude", pinyin: "claude") + ]) + } + + @Test func stripsLeadingByteOrderMark() { + #expect(VocabularyParser.parse("\u{feff}新词\n") == [VocabularyEntry(text: "新词", pinyin: "xin ci")]) + let parsed = VocabularyParser.parse("\u{feff}# comment\n---\nname: demo\ncolumns: [text, weight, code]\n...\n银行\t90\tyin hang\n") + #expect(parsed == [VocabularyEntry(text: "银行", pinyin: "yin hang", weight: 90)]) + } + + @Test(arguments: ["[text, code, code]", "[weight, code]", "[text, 1]", "[]", "text", "[text, code"]) + func rejectsInvalidColumns(columns: String) { + #expect(VocabularyParser.parse("---\nname: demo\ncolumns: \(columns)\n...\n银行\tyin hang\t90\n").isEmpty) + } + + @Test func rejectsAmbiguousOrIncompleteHeaders() { + let duplicate = "---\nname: demo\ncolumns: [text, code]\ncolumns: [code, text]\n...\n银行\tyin hang\n" + #expect(VocabularyParser.parse(duplicate).isEmpty) + #expect(VocabularyParser.parse("# comment\n---\nname: demo\ncolumns: [text, code]\n银行\tyin hang\n").isEmpty) + let oversized = "---\nname: demo\ndescription: " + String(repeating: "x", count: 64 * 1024) + "\n...\n银行\tyin hang\n" + #expect(VocabularyParser.parse(oversized).isEmpty) + } + + @Test func rebuildsExistingTablesWithTheCorrectedParser() throws { + let fm = FileManager.default + let root = fm.temporaryDirectory.appendingPathComponent("aime-parser-\(UUID().uuidString)") + defer { try? fm.removeItem(at: root) } + let paths = AIMEPaths(userDataDir: root) + let manager = SubscriptionManager(paths: paths) + let item = VocabularySubscription(id: "sample", name: "Sample", url: URL(string: "https://example.invalid/dict.yaml")!, addedAt: Date()) + try manager.save([item]) + let cache = manager.cacheURL(item.id) + try fm.createDirectory(at: cache.deletingLastPathComponent(), withIntermediateDirectories: true) + let content = Data("\u{feff}---\nname: demo\ncolumns: [text, weight, code]\n...\n银行\t20\tyin hang\n银行\t90\tyin hang\n".utf8) + try content.write(to: cache) + let modified = try cache.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate + try "schema_list:\n - schema: rime_ice\n".write(to: root.appendingPathComponent("default.yaml"), atomically: true, encoding: .utf8) + try manager.rebuild(schemas: ["rime_ice"]) + let table = root.appendingPathComponent("aime_online.txt") + try "银行\tyinxing\t50\n".write(to: table, atomically: true, encoding: .utf8) + let signature = manager.tableSignature(schemas: ["rime_ice"], items: [item]) + let oldSignature = "v2|" + signature.split(separator: "|", maxSplits: 1)[1] + try oldSignature.write(to: paths.aimeDir.appendingPathComponent("subscriptions/.signature"), atomically: true, encoding: .utf8) + + #expect(try manager.ensureTables()) + let rows = try String(contentsOf: table, encoding: .utf8).split(separator: "\n").filter { !$0.hasPrefix("#") } + #expect(rows == ["银行\tyinhang\t90"]) + #expect(try Data(contentsOf: cache) == content) + #expect(try cache.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate == modified) + #expect(try manager.ensureTables() == false) + } +}