diff --git a/README.md b/README.md index 2e8803b..1bcae6a 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ as both systems tackle the identical challenge to extract plain text from attach ## Installation +Requires ActiveSupport 6.0 or newer. + Add this line to your application's Gemfile: ```ruby @@ -111,6 +113,7 @@ fulltext = Plaintext::Resolver.new(file, content_type).text To limit the number of bytes returned (default is 4MB), set the `max_plaintext_bytes` property on the resolver instance before calling `text`. +Text is normalized to NFC, and byte limits preserve complete grapheme clusters. ## License diff --git a/lib/plaintext/resolver.rb b/lib/plaintext/resolver.rb index 8d84533..dd1b95d 100644 --- a/lib/plaintext/resolver.rb +++ b/lib/plaintext/resolver.rb @@ -38,7 +38,8 @@ def text if handler = find_handler and text = handler.text(@file, max_size: max_plaintext_bytes) - text.mb_chars.compose.limit(max_plaintext_bytes).to_s + text = text.dup.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8 + text.unicode_normalize(:nfc).truncate_bytes(max_plaintext_bytes, omission: nil) end end diff --git a/plaintext.gemspec b/plaintext.gemspec index d272779..08a8022 100644 --- a/plaintext.gemspec +++ b/plaintext.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'activesupport', '>2.2.1 ' + spec.add_dependency 'activesupport', '>= 6.0' spec.add_dependency 'nokogiri', '~> 1.10', '>= 1.10.4' spec.add_dependency 'rubyzip', '>= 1.2.0' diff --git a/spec/lib/file_handler/resolver_spec.rb b/spec/lib/file_handler/resolver_spec.rb index 6bbe90f..15f1d5c 100644 --- a/spec/lib/file_handler/resolver_spec.rb +++ b/spec/lib/file_handler/resolver_spec.rb @@ -31,4 +31,45 @@ expect(resolver.text).to include(/lorem ipsum/) end + it 'normalizes decomposed text without calling mb_chars' do + text = +"Cafe\u0301" + allow(handler).to receive(:text).and_return(text) + expect(text).not_to receive(:mb_chars) + + expect(resolver.text).to eq("Caf\u00e9") + end + + it 'limits bytes without returning a partial Unicode character' do + allow(handler).to receive(:text).and_return("ab\u00e9\u{1f600}z") + + { 0 => '', 1 => 'a', 3 => 'ab', 4 => "ab\u00e9", 7 => "ab\u00e9", 8 => "ab\u00e9\u{1f600}" }.each do |limit, expected| + resolver.max_plaintext_bytes = limit + expect(resolver.text).to eq(expected) + expect(resolver.text).to be_valid_encoding + end + end + + it 'rejects a negative byte limit' do + allow(handler).to receive(:text).and_return('abc') + resolver.max_plaintext_bytes = -1 + + expect { resolver.text }.to raise_error(ArgumentError) + end + + it 'treats extracted bytes as UTF-8 without mutating a frozen binary source' do + text = "Cafe\u0301".b.freeze + allow(handler).to receive(:text).and_return(text) + + expect(resolver.text).to eq("Caf\u00e9") + expect(resolver.text.encoding).to eq(Encoding::UTF_8) + expect(text.encoding).to eq(Encoding::BINARY) + end + + it 'keeps a complete grapheme when a byte limit falls inside a joined emoji' do + allow(handler).to receive(:text).and_return("ab\u{1f468}\u200d\u{1f469}\u200d\u{1f467}\u200d\u{1f466}") + resolver.max_plaintext_bytes = 8 + + expect(resolver.text).to eq('ab') + end + end