Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ as both systems tackle the identical challenge to extract plain text from attach

## Installation

Requires ActiveSupport 6.0 or newer.

Add this line to your application's Gemfile:

```ruby
Expand Down Expand Up @@ -111,6 +113,7 @@ fulltext = Plaintext::Resolver.new(file, content_type).text

To limit the number of bytes returned (default is 4MB), set the
`max_plaintext_bytes` property on the resolver instance before calling `text`.
Text is normalized to NFC, and byte limits preserve complete grapheme clusters.

## License

Expand Down
3 changes: 2 additions & 1 deletion lib/plaintext/resolver.rb
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def text
if handler = find_handler and
text = handler.text(@file, max_size: max_plaintext_bytes)

text.mb_chars.compose.limit(max_plaintext_bytes).to_s
text = text.dup.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8
text.unicode_normalize(:nfc).truncate_bytes(max_plaintext_bytes, omission: nil)
end
end

Expand Down
2 changes: 1 addition & 1 deletion plaintext.gemspec
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
spec.require_paths = ['lib']

spec.add_dependency 'activesupport', '>2.2.1 '
spec.add_dependency 'activesupport', '>= 6.0'
spec.add_dependency 'nokogiri', '~> 1.10', '>= 1.10.4'
spec.add_dependency 'rubyzip', '>= 1.2.0'

Expand Down
41 changes: 41 additions & 0 deletions spec/lib/file_handler/resolver_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -31,4 +31,45 @@

expect(resolver.text).to include(/lorem ipsum/)
end
it 'normalizes decomposed text without calling mb_chars' do
text = +"Cafe\u0301"
allow(handler).to receive(:text).and_return(text)
expect(text).not_to receive(:mb_chars)

expect(resolver.text).to eq("Caf\u00e9")
end

it 'limits bytes without returning a partial Unicode character' do
allow(handler).to receive(:text).and_return("ab\u00e9\u{1f600}z")

{ 0 => '', 1 => 'a', 3 => 'ab', 4 => "ab\u00e9", 7 => "ab\u00e9", 8 => "ab\u00e9\u{1f600}" }.each do |limit, expected|
resolver.max_plaintext_bytes = limit
expect(resolver.text).to eq(expected)
expect(resolver.text).to be_valid_encoding
end
end

it 'rejects a negative byte limit' do
allow(handler).to receive(:text).and_return('abc')
resolver.max_plaintext_bytes = -1

expect { resolver.text }.to raise_error(ArgumentError)
end

it 'treats extracted bytes as UTF-8 without mutating a frozen binary source' do
text = "Cafe\u0301".b.freeze
allow(handler).to receive(:text).and_return(text)

expect(resolver.text).to eq("Caf\u00e9")
expect(resolver.text.encoding).to eq(Encoding::UTF_8)
expect(text.encoding).to eq(Encoding::BINARY)
end

it 'keeps a complete grapheme when a byte limit falls inside a joined emoji' do
allow(handler).to receive(:text).and_return("ab\u{1f468}\u200d\u{1f469}\u200d\u{1f467}\u200d\u{1f466}")
resolver.max_plaintext_bytes = 8

expect(resolver.text).to eq('ab')
end

end
Loading