diff --git a/CODEX_HANDOFF.md b/CODEX_HANDOFF.md index 8176621..a66f98b 100644 --- a/CODEX_HANDOFF.md +++ b/CODEX_HANDOFF.md @@ -73,6 +73,15 @@ or temporary deployment credentials. focus, so externally downloaded files appear without a restart while collapsed branches stay inexpensive. Their context menu also supports the existing workspace file operations and opening through the system application. +- **Structured local document ingestion (2026-08-19):** Existing Word, + PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and text-based PDF files are + converted locally to structured Markdown with the pinned MIT-licensed anydoc + Rust library before reaching the Agent. This replaces TierNote's former light + OOXML/PDF text scraping, preserves document structure more faithfully, and + adds fixed parser resource limits. Scanned or image-only PDFs still require a + separate OCR path. Keep the native TierNote DOCX/PDF generators: anydoc is an + input parser, not an output-document generator. Do not invoke its npx skill or + bind the app to a developer-local checkout. - The current Agent tool surface can edit text files but cannot execute terminal commands. It must state that honestly and never claim builds or tests ran. Add shell execution only together with an explicit user approval and safety model. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index f99dcef..5f23d9e 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,5 +1,37 @@ # Third-party notices +## anydoc + +TierNote uses anydoc 0.1.9 to convert local Word, PowerPoint, Excel, +OpenDocument, RTF, EPUB, CSV, and text-based PDF files into structured Markdown +for the Agent. Conversion runs locally inside the desktop application; TierNote +does not use the hosted Firecrawl Parse service. + +MIT License + +Copyright (c) 2026 Sideguide Technologies Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Source and license: +https://github.com/firecrawl/anydoc/blob/v0.1.9/LICENSE + ## printpdf TierNote uses printpdf 0.7 to write PDF files and subset local TrueType fonts. diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 6c4e84e..f605be4 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -74,6 +85,73 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anydoc" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93a8b0dbecf79be9329111093b493d256db2a573ed3f1cc827c96bdbbe32a936" +dependencies = [ + "calamine", + "cfb 0.14.0", + "csv", + "encoding_rs", + "flate2", + "log", + "pdf-inspector", + "quick-xml", + "zip 8.6.0", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -270,6 +348,16 @@ dependencies = [ "system-deps", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -342,6 +430,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -473,6 +570,24 @@ dependencies = [ "system-deps", ] +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "chrono", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml", + "serde", + "zip 8.6.0", +] + [[package]] name = "camino" version = "1.2.5" @@ -515,6 +630,15 @@ dependencies = [ "toml 0.9.12+spec-1.1.0", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.4.0" @@ -542,6 +666,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "cfb" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a347dcabdae9c31b0825fd6a8bed285ec9c2acb89c47827126d52fa4f59cece3" +dependencies = [ + "fnv", + "uuid", + "web-time", +] + [[package]] name = "cfg-expr" version = "0.15.8" @@ -589,6 +724,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clipboard-win" version = "5.4.1" @@ -598,6 +743,21 @@ dependencies = [ "error-code", ] +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -703,6 +863,25 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.22" @@ -748,6 +927,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.8.0" @@ -809,6 +1009,43 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.19", +] + [[package]] name = "deranged" version = "0.5.8" @@ -999,6 +1236,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + [[package]] name = "either" version = "1.17.0" @@ -1061,6 +1307,29 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1131,6 +1400,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + [[package]] name = "fastrand" version = "2.5.0" @@ -1192,6 +1467,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1994,6 +2270,25 @@ dependencies = [ "tiff", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -2023,7 +2318,17 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" dependencies = [ - "cfb", + "cfb 0.7.3", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", ] [[package]] @@ -2051,6 +2356,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.10.5" @@ -2089,6 +2400,59 @@ dependencies = [ "system-deps", ] +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link 0.2.1", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.21.1" @@ -2291,6 +2655,37 @@ dependencies = [ "weezl", ] +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "chrono", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap 2.14.0", + "itoa", + "jiff", + "log", + "md-5", + "nom 8.0.0", + "rand", + "rangemap", + "rayon", + "sha2", + "stringprep", + "thiserror 2.0.19", + "time", + "ttf-parser 0.25.1", + "weezl", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2308,6 +2703,16 @@ dependencies = [ "web_atoms", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "md5" version = "0.7.0" @@ -2692,6 +3097,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "open" version = "5.4.0" @@ -2759,7 +3170,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "706de7e2214113d63a8238d1910463cfce781129a6f263d13fdb09ff64355ba4" dependencies = [ - "ttf-parser", + "ttf-parser 0.19.2", ] [[package]] @@ -2816,6 +3227,24 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "pdf-inspector" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d004766b3256499a00dc15c05d99872a24ab6d95ee34b6409eaaec4b793b3a" +dependencies = [ + "env_logger", + "include_dir", + "log", + "lopdf 0.42.0", + "once_cell", + "rayon", + "regex", + "thiserror 2.0.19", + "ttf-parser 0.25.1", + "unicode-normalization", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2971,6 +3400,21 @@ dependencies = [ "bstr", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2999,7 +3443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c30a4cc87c3ca9a98f4970db158a7153f8d1ec8076e005751173c57836380b1d" dependencies = [ "allsorts", - "lopdf", + "lopdf 0.31.0", "owned_ttf_parser", "time", ] @@ -3084,6 +3528,7 @@ version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ + "encoding_rs", "memchr", ] @@ -3190,12 +3635,38 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + [[package]] name = "raw-window-handle" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3931,6 +4402,17 @@ dependencies = [ "quote", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -4670,6 +5152,7 @@ dependencies = [ name = "tiernote" version = "0.1.8" dependencies = [ + "anydoc", "base64 0.22.1", "chrono", "dirs", @@ -4698,7 +5181,7 @@ dependencies = [ "tokio", "tokio-util", "uuid", - "zip", + "zip 2.4.2", ] [[package]] @@ -5061,6 +5544,18 @@ version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49d64318d8311fc2668e48b63969f4343e0a85c4a109aa8460d6672e364b8bd1" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typeid" version = "1.0.3" @@ -5137,6 +5632,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-canonical-combining-class" version = "0.5.0" @@ -5161,6 +5662,21 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f8cb47ccb8bc750808755af3071da4a10dcd147b68fc874b7ae4b12543f6f5" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -5210,6 +5726,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.24.0" @@ -6337,6 +6859,26 @@ dependencies = [ "zopfli", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap 2.14.0", + "memchr", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index c25092c..a04e26f 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -5,7 +5,7 @@ description = "Local-first workspace agent with tier-based organization and AI a authors = ["TierNote Team"] license = "AGPL-3.0-or-later" edition = "2021" -rust-version = "1.77.2" +rust-version = "1.88" [lib] name = "tiernote_lib" @@ -30,6 +30,7 @@ chrono = "0.4" futures = "0.3" uuid = { version = "1", features = ["v4"] } log = "0.4" +anydoc = "=0.1.9" tokio = { version = "1", features = ["sync", "rt", "macros", "time", "fs", "io-util", "process"] } reqwest-eventsource = "0.6" tokio-util = "0.7" diff --git a/src-tauri/src/agent_tools.rs b/src-tauri/src/agent_tools.rs index f835cb2..59a6a64 100644 --- a/src-tauri/src/agent_tools.rs +++ b/src-tauri/src/agent_tools.rs @@ -280,8 +280,8 @@ pub fn get_tool_definitions(availability: ToolAvailability) -> Vec { name: "read_local_file".into(), description: "Read the content of a local file outside the selected workspace when the \ user explicitly supplies its absolute path. Use this when the user asks you to \ - import a local document — a PDF, Word (.docx), PowerPoint (.pptx), Excel (.xlsx), \ - HTML file, plain text, or an image (for multimodal models). \ + import a local document — PDF, Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, \ + CSV, HTML, plain text, or an image (for multimodal models). \ Text-based files are read as text; images are returned as paths for vision. \ Follow the user's requested outcome; do not automatically turn the file into a note. \ The file path must be absolute, e.g. 'C:/Users/name/Downloads/report.pdf'." @@ -1475,10 +1475,7 @@ async fn exec_read_local_file(args: &Value, locale: &str) -> ToolResult { } }; let transcript = match crate::transcription::transcribe_local_media_file( - &file_path, - "api", - &config, - locale, + &file_path, "api", &config, locale, ) .await { @@ -1505,14 +1502,19 @@ async fn exec_read_local_file(args: &Value, locale: &str) -> ToolResult { success: false, output: format!( "The file '{}' has an unsupported format (.{}). \ - Supported: .txt .md .html .docx .pptx .xlsx .pdf, images, and audio/video.", + Supported: plain text/HTML; PDF; Word, PowerPoint, Excel, \ + OpenDocument, RTF, EPUB, and CSV documents; images; and audio/video.", content.label, content.extension ), } } }; if output.len() > 300_000 { - output.truncate(300_000); + let mut boundary = 300_000; + while !output.is_char_boundary(boundary) { + boundary -= 1; + } + output.truncate(boundary); output.push_str("\n…[truncated]"); } ToolResult { diff --git a/src-tauri/src/file_reader.rs b/src-tauri/src/file_reader.rs index 98aad0c..202b51a 100644 --- a/src-tauri/src/file_reader.rs +++ b/src-tauri/src/file_reader.rs @@ -1,13 +1,11 @@ // File Reader — read local files into text content for the Agent. // -// Principle: 不转格式，只送内容 (no conversion, only content). -// Text files are read as-is; office/PDF files get a light text extraction -// (no layout/table reconstruction — that is the Agent's job); images are +// Text files are read as-is. Office, OpenDocument, RTF, EPUB, CSV, and PDF +// files are converted locally to structured Markdown with anydoc. Images are // returned as paths for the multimodal model; audio/video goes through the // existing transcription pipeline. use std::fs; -use std::io::Read; use std::path::Path; /// Max bytes of extracted text we hand to the agent (protects context). @@ -54,8 +52,8 @@ pub fn read_file_content(path: &Path) -> Result { match extension.as_str() { // ── Plain text: zero conversion ── - "txt" | "md" | "markdown" | "text" | "log" | "csv" | "tsv" | "json" | "yaml" | "yml" - | "xml" | "toml" | "ini" | "conf" | "srt" | "vtt" => { + "txt" | "md" | "markdown" | "text" | "log" | "tsv" | "json" | "yaml" | "yml" | "xml" + | "toml" | "ini" | "conf" | "srt" | "vtt" => { let text = read_text_limited(path)?; Ok(FileContent { kind: ContentKind::Text, @@ -65,15 +63,10 @@ pub fn read_file_content(path: &Path) -> Result { extension, }) } - // ── Office documents: light text extraction ── - "docx" => read_docx(path, &label, &extension), - "pptx" => read_pptx(path, &label, &extension), - "xlsx" => read_xlsx(path, &label, &extension), - "pdf" => read_pdf(path, &label, &extension), - "doc" | "ppt" | "xls" => Err(format!( - "Legacy Office format '{extension}' is not supported. \ - Save it as .docx/.pptx/.xlsx (or text) and try again." - )), + // ── Documents: local structured Markdown conversion ── + "doc" | "docx" | "docm" | "odt" | "rtf" | "epub" | "pdf" | "ppt" | "pps" | "pot" + | "pptx" | "pptm" | "ppsx" | "ppsm" | "odp" | "xls" | "xlsx" | "xlsm" | "xlsb" | "ods" + | "csv" => read_document(path, &label, &extension), // ── HTML: reuse existing reader logic ── "html" | "htm" => read_html(path, &label, &extension), // ── Images: return path for multimodal ── @@ -121,97 +114,18 @@ fn read_text_limited(path: &Path) -> Result { Ok(String::from_utf8_lossy(&bytes).into_owned()) } -/// Read .docx by unzipping and pulling text from word/document.xml. -fn read_docx(path: &Path, label: &str, extension: &str) -> Result { - let xml = read_zip_entry(path, "word/document.xml")?; - let text = truncate(strip_xml_tags(&xml), MAX_EXTRACT_BYTES); - Ok(FileContent { - kind: ContentKind::Text, - text, - image_path: None, - label: label.to_string(), - extension: extension.to_string(), - }) -} - -/// Read .pptx by unzipping and pulling text from slide XML parts. -fn read_pptx(path: &Path, label: &str, extension: &str) -> Result { - let mut text = String::new(); - for slide in 1..=200 { - let entry = format!("ppt/slides/slide{slide}.xml"); - match read_zip_entry_opt(path, &entry)? { - Some(xml) => { - text.push_str(&format!("\n\n--- slide {slide} ---\n")); - text.push_str(&strip_xml_tags(&xml)); - } - None => break, - } - } - if text.trim().is_empty() { - return Err("The .pptx file contains no extractable text.".to_string()); - } - let text = truncate(text, MAX_EXTRACT_BYTES); - Ok(FileContent { - kind: ContentKind::Text, - text, - image_path: None, - label: label.to_string(), - extension: extension.to_string(), - }) -} - -/// Read .xlsx by unzipping and pulling text from sharedStrings + sheet rows. -fn read_xlsx(path: &Path, label: &str, extension: &str) -> Result { - let mut text = String::new(); - let mut shared: Vec = Vec::new(); - if let Some(xml) = read_zip_entry_opt(path, "xl/sharedStrings.xml")? { - shared = extract_shared_strings(&xml); - } - let mut sheet_index = 1; - loop { - let entry = format!("xl/worksheets/sheet{sheet_index}.xml"); - let Some(xml) = read_zip_entry_opt(path, &entry)? else { - break; - }; - if sheet_index > 1 { - text.push_str(&format!("\n\n--- sheet {sheet_index} ---\n")); - } - text.push_str(&extract_sheet_rows(&xml, &shared)); - sheet_index += 1; - if sheet_index > 50 { - break; - } - } - if text.trim().is_empty() { - return Err("The .xlsx file contains no extractable text.".to_string()); - } - let text = truncate(text, MAX_EXTRACT_BYTES); - Ok(FileContent { - kind: ContentKind::Text, - text, - image_path: None, - label: label.to_string(), - extension: extension.to_string(), - }) -} - -/// Read .pdf by extracting the text layer (no OCR — that is the Agent's job). -fn read_pdf(path: &Path, label: &str, extension: &str) -> Result { - let bytes = fs::read(path).map_err(|error| format!("Could not read PDF: {error}"))?; - if bytes.len() > 100 * 1024 * 1024 { - return Err("The PDF is too large (over 100 MB).".to_string()); - } - let text = truncate(extract_pdf_text(&bytes)?, MAX_EXTRACT_BYTES); - if text.trim().is_empty() { - return Err( - "The PDF has no extractable text layer — it may be a scanned document. \ - Ask the agent to read it as images instead." - .to_string(), - ); +/// Convert a supported document to LLM-ready structured Markdown locally. +fn read_document(path: &Path, label: &str, extension: &str) -> Result { + let markdown = anydoc::to_markdown(path) + .map_err(|error| format!("Could not convert '{label}' to Markdown: {error}"))?; + if markdown.trim().is_empty() { + return Err(format!( + "The .{extension} file contains no extractable text. It may require OCR." + )); } Ok(FileContent { kind: ContentKind::Text, - text, + text: truncate(markdown, MAX_EXTRACT_BYTES), image_path: None, label: label.to_string(), extension: extension.to_string(), @@ -242,269 +156,16 @@ fn read_html(path: &Path, label: &str, extension: &str) -> Result String { if text.len() > max { - text.truncate(max); + let mut boundary = max; + while !text.is_char_boundary(boundary) { + boundary -= 1; + } + text.truncate(boundary); text.push_str("\n…[truncated]"); } text } -/// Read a single entry from a zip file, erroring if absent. -fn read_zip_entry(path: &Path, entry: &str) -> Result { - match read_zip_entry_opt(path, entry)? { - Some(content) => Ok(content), - None => Err(format!("The file is missing the expected '{entry}' part.")), - } -} - -/// Read a single entry from a zip file; None if the entry does not exist. -fn read_zip_entry_opt(path: &Path, entry: &str) -> Result, String> { - let file = fs::File::open(path).map_err(|error| format!("Could not open file: {error}"))?; - let mut archive = - zip::ZipArchive::new(file).map_err(|error| format!("Not a valid Office file: {error}"))?; - let mut found = None; - for i in 0..archive.len() { - let mut item = archive - .by_index(i) - .map_err(|error| format!("Could not read archive entry: {error}"))?; - if item.name() == entry { - let mut buf = Vec::new(); - item.read_to_end(&mut buf) - .map_err(|error| format!("Could not read archive entry: {error}"))?; - found = Some(String::from_utf8_lossy(&buf).into_owned()); - break; - } - } - Ok(found) -} - -/// Strip XML tags, converting paragraph/section breaks to newlines. -fn strip_xml_tags(xml: &str) -> String { - let mut out = String::with_capacity(xml.len()); - let mut in_tag = false; - for c in xml.chars() { - if c == '<' { - in_tag = true; - continue; - } - if c == '>' { - in_tag = false; - continue; - } - if !in_tag { - out.push(c); - } - } - // Collapse runs of blank lines. - let mut result = String::with_capacity(out.len()); - let mut blank = 0; - for line in out.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() { - blank += 1; - if blank <= 1 { - result.push('\n'); - } - } else { - blank = 0; - result.push_str(trimmed); - result.push('\n'); - } - } - result -} - -fn extract_shared_strings(xml: &str) -> Vec { - let mut strings = Vec::new(); - let mut in_si = false; - let mut in_t = false; - let mut current = String::new(); - let mut idx = 0usize; - let bytes = xml.as_bytes(); - while idx < bytes.len() { - if bytes[idx] == b'<' { - let end = xml[idx..] - .find('>') - .map(|v| idx + v + 1) - .unwrap_or(bytes.len()); - let tag = &xml[idx..end]; - if tag.starts_with(" String { - let mut out = String::new(); - let mut in_cell = false; - let mut cell_shared = false; - let mut in_v = false; - let mut v_text = String::new(); - let mut in_row = false; - let mut row_count = 0; - let mut idx = 0usize; - let bytes = xml.as_bytes(); - while idx < bytes.len() { - if bytes[idx] == b'<' { - let end = xml[idx..] - .find('>') - .map(|v| idx + v + 1) - .unwrap_or(bytes.len()); - let tag = &xml[idx..end]; - if tag.starts_with(" 1 { - out.push('\n'); - } - } else if tag.starts_with("") { - in_cell = true; - cell_shared = tag.contains("t=\"s\""); - v_text.clear(); - } else if tag.starts_with("() - .ok() - .and_then(|index| shared.get(index)) - .map(|text| text.as_str()) - .unwrap_or(v_text.trim()) - } else { - v_text.trim() - }; - if !value.is_empty() { - if !out.ends_with('\n') && !out.is_empty() { - out.push_str(" | "); - } - out.push_str(value); - } - } - } else if tag.starts_with(" Result { - let mut out = String::new(); - let mut pos = 0usize; - while pos + 6 < bytes.len() { - let Some(rel) = find_subslice(&bytes[pos..], b"stream") else { - break; - }; - let stream_start = pos + rel + 6; - let mut content_start = stream_start; - if content_start < bytes.len() && bytes[content_start] == b'\r' { - content_start += 1; - } - if content_start < bytes.len() && bytes[content_start] == b'\n' { - content_start += 1; - } - let Some(end_rel) = find_subslice(&bytes[content_start..], b"endstream") else { - break; - }; - let content = &bytes[content_start..content_start + end_rel]; - if !contains_subslice(content, b"FlateDecode") { - out.push_str(&extract_text_ops(content)); - } - pos = content_start + end_rel + 9; - } - Ok(out) -} - -fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool { - find_subslice(haystack, needle).is_some() -} - -/// Extract text from PDF content-stream operators: (text) Tj and [(a)(b)] TJ. -fn extract_text_ops(content: &[u8]) -> String { - let mut out = String::new(); - let mut idx = 0usize; - while idx < content.len() { - if content[idx] == b'(' { - let start = idx + 1; - let mut end = start; - let mut escaped = false; - while end < content.len() { - let c = content[end]; - if c == b'\\' && !escaped { - escaped = true; - end += 1; - continue; - } - if c == b')' && !escaped { - break; - } - escaped = false; - end += 1; - } - out.push_str(&String::from_utf8_lossy(&content[start..end])); - idx = end + 1; - } else if content[idx] == b'[' { - let mut end = idx + 1; - let mut depth = 1; - while end < content.len() && depth > 0 { - if content[end] == b'[' { - depth += 1; - } else if content[end] == b']' { - depth -= 1; - } - end += 1; - } - idx = end; - } else if content[idx] == b'<' { - let mut end = idx + 1; - while end < content.len() && content[end] != b'>' { - end += 1; - } - idx = end + 1; - } else { - idx += 1; - } - } - out -} - fn path_string(path: &Path) -> String { path.to_string_lossy().into_owned() } @@ -541,6 +202,31 @@ mod tests { assert!(result.text.contains("Hello World"), "got: {}", result.text); } + #[test] + fn csv_converts_to_a_markdown_table() { + let path = temp_dir("csv").join("sample.csv"); + fs::write(&path, "name,score\nAlpha,1\nBeta,2\n").unwrap(); + let result = read_file_content(&path).unwrap(); + assert_eq!(result.kind, ContentKind::Text); + assert!( + result.text.contains("| name | score |"), + "got: {}", + result.text + ); + assert!( + result.text.contains("| Alpha | 1 |"), + "got: {}", + result.text + ); + } + + #[test] + fn markdown_truncation_stays_on_a_utf8_boundary() { + let text = truncate("中文内容".repeat(100), 17); + assert!(text.ends_with("…[truncated]")); + assert!(text.is_char_boundary(text.len())); + } + #[test] fn image_returns_path() { let path = temp_dir("img").join("photo.png"); @@ -564,14 +250,6 @@ mod tests { assert!(result.is_err()); } - #[test] - fn strip_tags_converts_breaks() { - let xml = "HelloWorld"; - let text = strip_xml_tags(xml); - assert!(text.contains("Hello")); - assert!(text.contains("World")); - } - fn build_mini_docx(path: &Path) { let file = fs::File::create(path).unwrap(); let mut zip = zip::ZipWriter::new(file); diff --git a/src-tauri/src/knowledge_map.rs b/src-tauri/src/knowledge_map.rs index 870a093..5b3c3ae 100644 --- a/src-tauri/src/knowledge_map.rs +++ b/src-tauri/src/knowledge_map.rs @@ -245,7 +245,7 @@ fn retrieve_context_scoped( .filter_map(|(note_index, note)| { let allowed = explicitly_allowed .as_ref() - .map_or(true, |indices| indices.contains(¬e_index)); + .is_none_or(|indices| indices.contains(¬e_index)); let excluded = excluded_prefixes .iter() .any(|prefix| path_has_prefix(¬e.path, prefix)); diff --git a/src-tauri/src/presentation.rs b/src-tauri/src/presentation.rs index 4233b8c..7bcd4d0 100644 --- a/src-tauri/src/presentation.rs +++ b/src-tauri/src/presentation.rs @@ -246,9 +246,10 @@ fn validate_slide(mut slide: PresentationSlide, index: usize) -> Result estimated_lines(&slide.body, 52) <= 17, "content" => estimated_lines(&slide.body, 86) <= 18, - "quote" => slide.body.first().map_or(true, |quote| { - estimated_lines(std::slice::from_ref(quote), 64) <= 7 - }), + "quote" => slide + .body + .first() + .is_none_or(|quote| estimated_lines(std::slice::from_ref(quote), 64) <= 7), _ => true, }; if !fits { diff --git a/src/App.tsx b/src/App.tsx index ef0ec53..165a9aa 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -4231,12 +4231,15 @@ const VIDEO_FILE_EXTENSIONS = new Set(['mp4', 'mov', 'mkv', 'avi', 'webm', 'm4v' const AUDIO_FILE_EXTENSIONS = new Set(['mp3', 'wav', 'm4a', 'aac', 'flac', 'ogg', 'opus', 'wma']); const IMAGE_FILE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'svg', 'heic', 'avif']); const ARCHIVE_FILE_EXTENSIONS = new Set(['zip', 'rar', '7z', 'tar', 'gz', 'bz2', 'xz']); -const SPREADSHEET_FILE_EXTENSIONS = new Set(['csv', 'tsv', 'xls', 'xlsx', 'ods']); +const SPREADSHEET_FILE_EXTENSIONS = new Set(['csv', 'tsv', 'xls', 'xlsx', 'xlsm', 'xlsb', 'ods']); const CODE_FILE_EXTENSIONS = new Set([ 'js', 'jsx', 'ts', 'tsx', 'py', 'rs', 'go', 'java', 'c', 'cc', 'cpp', 'h', 'hpp', 'html', 'css', 'scss', 'json', 'yaml', 'yml', 'toml', 'xml', 'sh', 'ps1', 'sql', ]); -const DOCUMENT_FILE_EXTENSIONS = new Set(['txt', 'log', 'rtf', 'pdf', 'doc', 'docx', 'ppt', 'pptx', 'odt']); +const DOCUMENT_FILE_EXTENSIONS = new Set([ + 'txt', 'log', 'rtf', 'pdf', 'epub', 'doc', 'docx', 'docm', 'odt', + 'ppt', 'pps', 'pot', 'pptx', 'pptm', 'ppsx', 'ppsm', 'odp', +]); function workspaceFileIcon( path: string, @@ -7573,7 +7576,7 @@ function ConversationView({

{locale === 'zh' - ? 'DeepSeek 高效缓存引擎 · 双向记忆路由 · Library Graph · 自动压缩' + ? '高效缓存引擎 · 双向记忆路由 · Library Graph · 自动压缩' : 'DeepSeek cache optimization · bidirectional memory routing · Library Graph · automatic compaction'}