From a41c4b77c63b25741f2cc5d1da4fe187ef3a1bd9 Mon Sep 17 00:00:00 2001 From: benjaaguila Date: Thu, 15 Oct 2026 20:43:30 -0300 Subject: [PATCH 1/2] fix: conservar texto unicode sin usar APIs obsoletas de Rails Mantener normalizacion NFC y truncamiento por bytes sin cortar caracteres. Rails 8.1 depreca mb_chars y Multibyte::Chars, usados al extraer documentos. --- lib/plaintext/resolver.rb | 4 +++- spec/lib/file_handler/resolver_spec.rb | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/plaintext/resolver.rb b/lib/plaintext/resolver.rb index 8d84533..437d3e6 100644 --- a/lib/plaintext/resolver.rb +++ b/lib/plaintext/resolver.rb @@ -38,7 +38,9 @@ def text if handler = find_handler and text = handler.text(@file, max_size: max_plaintext_bytes) - text.mb_chars.compose.limit(max_plaintext_bytes).to_s + raise ArgumentError, 'max_plaintext_bytes must be non-negative' if max_plaintext_bytes < 0 + + text.unicode_normalize(:nfc).byteslice(0, max_plaintext_bytes).scrub('') end end diff --git a/spec/lib/file_handler/resolver_spec.rb b/spec/lib/file_handler/resolver_spec.rb index 6bbe90f..650e854 100644 --- a/spec/lib/file_handler/resolver_spec.rb +++ b/spec/lib/file_handler/resolver_spec.rb @@ -31,4 +31,29 @@ expect(resolver.text).to include(/lorem ipsum/) end + it 'normalizes decomposed text without calling mb_chars' do + text = +"Cafe\u0301" + allow(handler).to receive(:text).and_return(text) + expect(text).not_to receive(:mb_chars) + + expect(resolver.text).to eq("Caf\u00e9") + end + + it 'limits bytes without returning a partial Unicode character' do + allow(handler).to receive(:text).and_return("ab\u00e9\u{1f600}z") + + { 0 => '', 1 => 'a', 3 => 'ab', 4 => "ab\u00e9", 7 => "ab\u00e9", 8 => "ab\u00e9\u{1f600}" }.each do |limit, expected| + resolver.max_plaintext_bytes = limit + expect(resolver.text).to eq(expected) + expect(resolver.text).to be_valid_encoding + end + end + + it 'rejects a negative byte limit' do + allow(handler).to receive(:text).and_return('abc') + resolver.max_plaintext_bytes = -1 + + expect { resolver.text }.to raise_error(ArgumentError) + end + end From e61d7312a9ed844a094b34312e7fe01849a9f342 Mon Sep 17 00:00:00 2001 From: benjaaguila Date: Thu, 15 Oct 2026 21:19:05 -0300 Subject: [PATCH 2/2] fix: preservar codificacion y grafemas al limitar texto extraido Conservar la conversion a UTF-8 de los bytes de los handlers y truncar mediante String#truncate_bytes sin cortar grafemas. Declarar ActiveSupport 6.0 como minimo para la API utilizada y cubrir ambos contratos sin invocar mb_chars. --- README.md | 3 +++ lib/plaintext/resolver.rb | 5 ++--- plaintext.gemspec | 2 +- spec/lib/file_handler/resolver_spec.rb | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2e8803b..1bcae6a 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ as both systems tackle the identical challenge to extract plain text from attach ## Installation +Requires ActiveSupport 6.0 or newer. + Add this line to your application's Gemfile: ```ruby @@ -111,6 +113,7 @@ fulltext = Plaintext::Resolver.new(file, content_type).text To limit the number of bytes returned (default is 4MB), set the `max_plaintext_bytes` property on the resolver instance before calling `text`. +Text is normalized to NFC, and byte limits preserve complete grapheme clusters. ## License diff --git a/lib/plaintext/resolver.rb b/lib/plaintext/resolver.rb index 437d3e6..dd1b95d 100644 --- a/lib/plaintext/resolver.rb +++ b/lib/plaintext/resolver.rb @@ -38,9 +38,8 @@ def text if handler = find_handler and text = handler.text(@file, max_size: max_plaintext_bytes) - raise ArgumentError, 'max_plaintext_bytes must be non-negative' if max_plaintext_bytes < 0 - - text.unicode_normalize(:nfc).byteslice(0, max_plaintext_bytes).scrub('') + text = text.dup.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8 + text.unicode_normalize(:nfc).truncate_bytes(max_plaintext_bytes, omission: nil) end end diff --git a/plaintext.gemspec b/plaintext.gemspec index d272779..08a8022 100644 --- a/plaintext.gemspec +++ b/plaintext.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activesupport', '>2.2.1 ' + spec.add_dependency 'activesupport', '>= 6.0' spec.add_dependency 'nokogiri', '~> 1.10', '>= 1.10.4' spec.add_dependency 'rubyzip', '>= 1.2.0' diff --git a/spec/lib/file_handler/resolver_spec.rb b/spec/lib/file_handler/resolver_spec.rb index 650e854..15f1d5c 100644 --- a/spec/lib/file_handler/resolver_spec.rb +++ b/spec/lib/file_handler/resolver_spec.rb @@ -56,4 +56,20 @@ expect { resolver.text }.to raise_error(ArgumentError) end + it 'treats extracted bytes as UTF-8 without mutating a frozen binary source' do + text = "Cafe\u0301".b.freeze + allow(handler).to receive(:text).and_return(text) + + expect(resolver.text).to eq("Caf\u00e9") + expect(resolver.text.encoding).to eq(Encoding::UTF_8) + expect(text.encoding).to eq(Encoding::BINARY) + end + + it 'keeps a complete grapheme when a byte limit falls inside a joined emoji' do + allow(handler).to receive(:text).and_return("ab\u{1f468}\u200d\u{1f469}\u200d\u{1f467}\u200d\u{1f466}") + resolver.max_plaintext_bytes = 8 + + expect(resolver.text).to eq('ab') + end + end