diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index e66d198..721b229 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -11,13 +11,14 @@ jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.version == 'nightly' }} strategy: fail-fast: false matrix: version: - '1.6' - '1' # automatically expands to the latest stable 1.x release of Julia - - nightly + - 'nightly' os: - ubuntu-latest arch: diff --git a/.gitignore b/.gitignore index a8deefa..76b8f77 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ test/*.pdf test/*.res test/pvt test/PDFTest* +test/pdftest* file.txt data/fonts/Arial.afm test/sample01.pem diff --git a/Project.toml b/Project.toml index f3154fd..170c7e9 100644 --- a/Project.toml +++ b/Project.toml @@ -31,6 +31,7 @@ julia = "1.6" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" [targets] -test = ["Test", "ZipFile"] +test = ["Test", "ZipFile", "Downloads"] diff --git a/src/CosDoc.jl b/src/CosDoc.jl index 7a979b5..414b4a9 100644 --- a/src/CosDoc.jl +++ b/src/CosDoc.jl @@ -391,7 +391,7 @@ function read_trailer(ps::IOStream, lookahead::Int) end function doc_trailer_update(ps::IOStream, doc::CosDocImpl) - TRAILER_REWIND = 50 + TRAILER_REWIND = 256 seek(ps, doc.size-TRAILER_REWIND) diff --git a/src/Inflate.jl b/src/Inflate.jl index b67d367..3959f86 100644 --- a/src/Inflate.jl +++ b/src/Inflate.jl @@ -45,13 +45,7 @@ const Z_MEM_ERROR = -4 const Z_BUF_ERROR = -5 const Z_VERSION_ERROR = -6 -@static if Base.VERSION > v"1.3-" - using Zlib_jll: libz -else - isfile(joinpath(dirname(@__FILE__),"..","deps","deps.jl")) || - error("PDFIO not properly installed. Please run Pkg.build(\"PDFIO\")") - include("../deps/deps.jl") -end +using Zlib_jll: libz _zlibVersion() = ccall((:zlibVersion, libz), Ptr{Cstring}, ()) diff --git a/src/LibCrypto.jl b/src/LibCrypto.jl index 05a340a..a8cce65 100644 --- a/src/LibCrypto.jl +++ b/src/LibCrypto.jl @@ -1,10 +1,4 @@ -@static if Base.VERSION > v"1.3-" - using OpenSSL_jll: libcrypto -else - isfile(joinpath(dirname(@__FILE__), "..", "deps", "deps.jl")) || - error("PDFIO not properly installed. Please run Pkg.build(\"PDFIO\")") - include("../deps/deps.jl") -end +using OpenSSL_jll: libcrypto using Base: SecretBuffer, SecretBuffer! import Base: copy diff --git a/src/PDFontTables.jl b/src/PDFontTables.jl index 9052111..0848735 100644 --- a/src/PDFontTables.jl +++ b/src/PDFontTables.jl @@ -55,15 +55,17 @@ const GlyphName_to_ZAPEncoding = reverse_dict(ZAPEncoding_to_GlyphName) using AdobeGlyphList -function agl_mapping_to_dict(m) +function agl_mapping_to_dict(m; fn=false) dict = Dict{CosName, Char}() - map((@view m[:,1]), (@view m[:,2])) do x, y + v1, v2 = fn ? (2, 1) : (1, 2) + map((@view m[:,v1]), (@view m[:,v2])) do x, y dict[CosName(strip(x))] = y end return dict end const AGL_Glyph_to_Unicode = agl_mapping_to_dict(agl()) +const AGLFN_Glyph_to_Unicode = agl_mapping_to_dict(aglfn(), fn=true) const AGL_ZAP_to_Unicode = agl_mapping_to_dict(zapfdingbats()) const AGL_Unicode_to_Glyph = reverse_dict(AGL_Glyph_to_Unicode) const AGL_Unicode_to_ZAP = reverse_dict(AGL_ZAP_to_Unicode) diff --git a/src/PDFonts.jl b/src/PDFonts.jl index 94399df..89feb04 100644 --- a/src/PDFonts.jl +++ b/src/PDFonts.jl @@ -22,6 +22,14 @@ const endbfrange = b"endbfrange" const begincodespacerange = b"begincodespacerange" const endcodespacerange = b"endcodespacerange" +# EXTRACTION_MODE determines how text extraction of textboxes is handled +# - `:spaces` (default) +# all white spaces are handled as a single space character +# - `:tabs` +# non-space white spaces are handled as tab characters +# - `:boxes` +# text is split into several textboxes with respective coordinates +const EXTRACTION_MODE = Ref(:spaces) # :spaces, :tabs, :boxes mutable struct CMap code_space::IntervalTree{UInt8, @@ -45,7 +53,7 @@ function show(io::IO, cmap::CMap) end -const FontUnicodeMapping = Union{Dict{UInt8, Char}, CMap, Nothing} +const FontUnicodeMapping = Union{Dict{UInt8, Vector{Char}}, CMap, Nothing} #= mutable struct FontUnicodeMapping @@ -56,7 +64,56 @@ mutable struct FontUnicodeMapping end =# -function merge_encoding!(fum::Dict{UInt8, Char}, encoding::CosName, +function Base.merge!(fum::Dict{UInt8, Vector{Char}}, enc::Dict{UInt8, Char}) + for (k, v) in enc + fum[k] = [v] + end +end + +function get_agl_unicode(g::AbstractString)::Union{Vector{Char}, Char} + r = r"u(?'u'[[:xdigit:]]+$)|uni(?'uni'[[:xdigit:]]{4,6}$)" + m = match(r, g) + if m !== nothing + u, uni = m["u"], m["uni"] + if u !== nothing + l = length(u) + if l > 3 && mod(l, 4) == 0 + ret = Char[] + for i = 1:4:l + c = parse(UInt16, SubString(u, i, i+3), base=16) + 0xE000 > c > 0xD7FF && break + push!(ret, Char(c)) + end + length(ret)*4 == l && return(ret) + end + else + c = parse(UInt32, uni, base=16) + 0x0000 <= c <= 0xD7FF && 0xE000 <= c <= 0x10FFFF && return Char(c) + end + end + cg = CosName(g) + return get(AGL_Glyph_to_Unicode, cg, get(AGLFN_Glyph_to_Unicode, cg, zero(Char))) +end + +function get_unicodes_from_glyph_name(s::String) + n = split(s, ".") + nf = n[1] + isempty(nf) && return [zero(Char)] + gs = split(nf, "_") + u = Char[] + for g in gs + append!(u, get_agl_unicode(g)) + end + return u +end + +function merge_agl!(fum::Dict{UInt8, Vector{Char}}, d::Dict{UInt8, CosName}) + for (k, v) in d + fum[k] = get_unicodes_from_glyph_name(String(v)) + end +end + +function merge_encoding!(fum::Dict{UInt8, Vector{Char}}, encoding::CosName, doc::CosDoc, font::IDDRef{CosDict}) encoding_mapping = encoding == cn"WinAnsiEncoding" ? WINEncoding_to_Unicode : @@ -82,10 +139,12 @@ function FontType(subtype::CosName) return FontDefType() end +# Entry point if someone wants to handle encoding based on subtype +# By default maps to the default font unicode mapping. merge_encoding!(fum::FontUnicodeMapping, ftype::FontType, doc::CosDoc, font::IDDRef{CosDict}) = fum -function merge_encoding!(fum::Dict{UInt8, Char}, +function merge_encoding!(fum::Dict{UInt8, Vector{Char}}, ftype::Union{FontType1, FontMMType1}, doc::CosDoc, font::IDDRef{CosDict}) basefont = cosDocGetObject(doc, font, cn"BaseFont") @@ -104,14 +163,14 @@ end # Reading encoding from the font files in case of Symbolic fonts are not # supported. # Font subset is addressed with font name identification. -function merge_encoding!(fum::Dict{UInt8, Char}, encoding::CosNullType, +function merge_encoding!(fum::Dict{UInt8, Vector{Char}}, encoding::CosNullType, doc::CosDoc, font::IDDRef{CosDict}) subtype = cosDocGetObject(doc, font, cn"Subtype") subtype === CosNull && return fum return merge_encoding!(fum, FontType(subtype), doc, font) end -function merge_encoding!(fum::Dict{UInt8, Char}, +function merge_encoding!(fum::Dict{UInt8, Vector{Char}}, encoding::IDD{CosDict}, doc::CosDoc, font::IDDRef{CosDict}) baseenc = cosDocGetObject(doc, encoding, cn"BaseEncoding") @@ -133,8 +192,7 @@ function merge_encoding!(fum::Dict{UInt8, Char}, end end - dict_to_unicode = dict_remap(d, AGL_Glyph_to_Unicode) - merge!(fum, dict_to_unicode) + merge_agl!(fum, d) return fum end @@ -143,7 +201,7 @@ function get_unicode_mapping(doc::CosDoc, font::IDDRef{CosDict}) toUnicode !== CosNull && return get_unicode_mapping(toUnicode) encoding = cosDocGetObject(doc, font, cn"Encoding") - d = merge_encoding!(Dict{UInt8, Char}(), encoding, doc, font) + d = merge_encoding!(Dict{UInt8, Vector{Char}}(), encoding, doc, font) return length(d) == 0 ? nothing : d end @@ -218,11 +276,11 @@ function get_glyph_id_mapping(cosdoc::CosDoc, cosfont::IDD{CosDict}) return glyph_name_to_cid, cid_to_glyph_name end -get_encoded_string(s::CosString, fum::Union{Dict{UInt8, Char}, CMap}) = +get_encoded_string(s::CosString, fum::FontUnicodeMapping) = get_encoded_string(Vector{UInt8}(s), fum) function get_encoded_string(v::Union{Vector{UInt8}, NTuple{N, UInt8}}, - fum::Dict{UInt8, Char}) where N + fum::Dict{UInt8, Vector{Char}}) where N length(v) == 0 && return "" return String(NativeEncodingToUnicode(v, fum)) end @@ -334,8 +392,17 @@ cmap_command(b::Vector{UInt8}) = length(b), b != beginbfchar && b != beginbfrange && b != begincodespacerange ? nothing : Symbol(String(b)) +function _offset(obj::CosXString, offset) + da = Vector{UInt8}(obj) + db = UInt16(da[1]*256+da[2]+offset) + da[1], da[2] = UInt8(div(db, 256)), UInt8(mod(db, 256)) + io = IOBuffer() + bytes2hex(io, da) + return CosXString(take!(io)) +end + function on_cmap_command!(stm::IO, command::Symbol, - params::Vector{CosInt}, cmap::CMap) + params::Vector{CosInt}, cmap::CMap) n = get(pop!(params)) o1, o2, o3 = CosNull, CosNull, CosNull for i = 1:n @@ -352,18 +419,57 @@ function on_cmap_command!(stm::IO, command::Symbol, if l == 1 cmap.range_map[Interval(d1[1], d2[1])] = o3 else - imap = get!(cmap.range_map, Interval(d1[1], d2[1]), - IntervalTree{UInt8, CosObject}()) - imap[Interval(d1[2], d2[2])] = o3 + if d1[2] <= d2[2] + imap = get!(cmap.range_map, Interval(d1[1], d2[1]), + IntervalTree{UInt8, CosObject}()) + imap[Interval(d1[2], d2[2])] = o3 + else + @warn "Corrupt CMap file. Repairing... Some encodings may not map properly." + imap = get!(cmap.range_map, Interval(d1[1], d1[1]), + IntervalTree{UInt8, CosObject}()) + imap[Interval(d1[2], 0xff)] = o3 + o3 = _offset(o3, 0xff - d1[2] + 1) + + if d2[1] - d1[1] > 1 + i1, i2 = d1[1]+0x1, d2[1]-0x1 + imap = get!(cmap.range_map, Interval(i1, i2), + IntervalTree{UInt8, CosObject}()) + imap[Interval(0x00, 0xff)] = o3 + o3 = _offset(o3, (d2[1] - d1[1] - 1)*0x100) + end + imap = get!(cmap.range_map, Interval(d2[1], d2[1]), + IntervalTree{UInt8, CosObject}()) + imap[Interval(0x00, d2[2])] = o3 + end end else l = length(d1) + @assert (d1[1] <= d2[1]) E_INVALID_CODESPACERANGE if l == 1 cmap.code_space[Interval(d1[1], d2[1])] = CosNull else - imap = IntervalTree{UInt8, CosNullType}() - imap[Interval(d1[2], d2[2])] = CosNull - cmap.code_space[Interval(d1[1], d2[1])] = imap + if d1[2] <= d2[2] + imap = IntervalTree{UInt8, CosNullType}() + imap[Interval(d1[2], d2[2])] = CosNull + cmap.code_space[Interval(d1[1], d2[1])] = imap + else + @warn "Corrupt CMap file. Repairing... Some encodings may not map properly." + imap = IntervalTree{UInt8, CosNullType}() + imap[Interval(d1[2], 0xff)] = CosNull + cmap.code_space[Interval(d1[1], d1[1])] = imap + + imap = get!(cmap.code_space, Interval(d1[1], d1[1]), IntervalTree{UInt8, CosNullType}()) + imap[Interval(d1[2], 0xff)] = CosNull + + imap = get!(cmap.code_space, Interval(d2[1], d2[1]), IntervalTree{UInt8, CosNullType}()) + imap[Interval(0x00, d2[2])] = CosNull + + if d2[1] - d1[1] > 1 + i1, i2 = d1[1]+0x1, d2[1]-0x1 + imap = get!(cmap.code_space, Interval(i1, i2), IntervalTree{UInt8, CosNullType}()) + imap[Interval(0x00, 0xff)] = CosNull + end + end end end end @@ -571,13 +677,26 @@ function get_TextBox(ss::Vector{Union{CosXString, CosLiteralString, totalw = 0f0 tj = 0f0 text = "" + offset = 0f0 + params = Tuple{String, Float32, Float32, Float32}[] for s in ss if s isa CosXString || s isa CosLiteralString prev_char = INIT_CODE(pdfont.widths) t = String(get_encoded_string(s, pdfont)) - if (-tj) > 180 && length(t) > 0 && t[1] != ' ' && - length(text) > 0 && text[end] != ' ' - text *= " " + if (-tj) > 180 + if EXTRACTION_MODE[] == :spaces + if length(t) > 0 && t[1] != ' ' && length(text) > 0 && text[end] != ' ' + text *= " " + end + elseif EXTRACTION_MODE[] == :tabs + text *= "\t" + elseif EXTRACTION_MODE[] == :boxes + push!(params, (text, totalw * th, tfs, offset * th)) + offset += totalw - tj * tfs / 1000f0 + text = "" + tj = 0f0 + totalw = 0f0 + end end text *= t barr = Vector{UInt8}(s) @@ -588,8 +707,8 @@ function get_TextBox(ss::Vector{Union{CosXString, CosLiteralString, tj = s |> get |> Float32 end end - totalw *= th - return text, totalw, tfs + push!(params, (text, totalw * th, tfs, offset * th)) + return params end function get_character_width(cid::UInt16, w::CIDWidth) diff --git a/src/PDPageElement.jl b/src/PDPageElement.jl index d042345..ca9d5f6 100644 --- a/src/PDPageElement.jl +++ b/src/PDPageElement.jl @@ -693,20 +693,27 @@ end fontname, font = eval_unicode_mapping(tr, state) heap = get(state, :text_layout, Vector{TextLayout}) - text, w, h = get_TextBox(tr.ss, font, tfs, tc, tw, th) + boxparams = get_TextBox(tr.ss, font, tfs, tc, tw, th) + h = boxparams[1][3] d = get(state, :h_profile, Dict{Int, Int}) ih = round(Int, h*10) - d[ih] = get(d, ih, 0) + length(text) - - tb = [0f0 0f0 1f0; w 0f0 1f0; w h 1f0; 0f0 h 1f0]*trm + if !get(state, :in_artifact, false) - tl = TextLayout(tb[1,1], tb[1,2], tb[2,1], tb[2,2], - tb[3,1], tb[3,2], tb[4,1], tb[4,2], - text, fontname, font.flags) - push!(heap, tl) + for params in boxparams + text = params[1] + w = params[2] + offset = params[4] + d[ih] = get(d, ih, 0) + length(text) + tb = [offset 0f0 1f0; offset + w 0f0 1f0; offset + w h 1f0; offset h 1f0]*trm + tl = TextLayout(tb[1,1], tb[1,2], tb[2,1], tb[2,2], + tb[3,1], tb[3,2], tb[4,1], tb[4,2], + text, fontname, font.flags) + push!(heap, tl) + end end - offset_text_pos!(w, 0f0, state) + totalw = boxparams[end][4] + boxparams[end][2] + offset_text_pos!(totalw, 0f0, state) return state end diff --git a/src/Utils.jl b/src/Utils.jl index 7fa6261..7203c24 100644 --- a/src/Utils.jl +++ b/src/Utils.jl @@ -80,10 +80,9 @@ end const Unicode_to_PDFEncoding = reverse_dict(PDFEncoding_to_Unicode) function NativeEncodingToUnicode(barr, mapping::Dict) - l = length(barr) - carr = Vector{Char}(undef, l) - for i = 1:l - carr[i] = get(mapping, barr[i], zero(Char)) + carr = Vector{Char}() + for b in barr + append!(carr, get(mapping, b, zero(Char))) end return carr end diff --git a/src/errors.jl b/src/errors.jl index 1ee01ff..5ae56a0 100644 --- a/src/errors.jl +++ b/src/errors.jl @@ -2,7 +2,7 @@ export E_EXPECTED_EOF, E_UNEXPECTED_EOF, E_UNEXPECTED_CHAR, E_BAD_KEY, E_BAD_ESCAPE, E_BAD_CONTROL, E_LEADING_ZERO, E_BAD_NUMBER, E_BAD_HEADER, E_BAD_TRAILER, E_BAD_TYPE, E_NOT_IMPLEMENTED, E_INVALID_OBJECT, E_INVALID_PAGE_NUMBER, E_INVALID_PAGE_LABEL, - E_NOT_TAGGED_PDF, E_INVALID_PASSWORD + E_NOT_TAGGED_PDF, E_INVALID_PASSWORD, E_INVALID_CODESPACERANGE # The following errors may be thrown by the reader const E_EXPECTED_EOF = "Expected end of input" @@ -28,3 +28,4 @@ const E_INVALID_PASSWORD = "The password supplied to open the document is inv const E_INVALID_CRYPT = "The crypt handler is not supported" const E_NOT_TAGGED_PDF = "PDF file is not tagged" const E_NOT_IMPLEMENTED = "Not Implemented" +const E_INVALID_CODESPACERANGE = "Invalid code space range in CMap" \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index a527d6a..4bdf6d8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,15 +5,16 @@ using PDFIO.Cos using PDFIO.Common using ZipFile using AbstractTrees +using Downloads # Internal methods for testing only using PDFIO.Cos: parse_indirect_ref, decode_ascii85, CosXString, parse_value -using PDFIO.PD: openssl_error +using PDFIO.PD: openssl_error, read_cmap, get_encoded_string using PDFIO.Common: read_pkcs12 include("debugIO.jl") -pdftest_ver = "0.0.9" +pdftest_ver = "0.0.11" pdftest_link = "https://github.com/sambitdash/PDFTest/archive/v"*pdftest_ver zipfile = "pdftest-"*pdftest_ver @@ -22,7 +23,7 @@ zipfile *= ".zip" pdftest_dir="PDFTest-"*pdftest_ver*"/" if !isdir(pdftest_dir) - isfile(zipfile) || download(pdftest_link, zipfile) + isfile(zipfile) || Downloads.download(pdftest_link, zipfile) r = ZipFile.Reader(zipfile) buf = Vector{UInt8}(undef, 64*1024) for f in r.files @@ -57,6 +58,8 @@ function local_testfiles(filename, filesdir="files") joinpath(@__DIR__, pdftest_dir, filesdir, filename)) end +local_files(filename, filesdir="files") = joinpath(@__DIR__, pdftest_dir, filesdir, filename) + @testset "PDFIO tests" begin @testset "Miscellaneous" begin @test_throws ErrorException skipv(IOBuffer([UInt8(65), UInt8(66)]), @@ -416,6 +419,41 @@ end @test files_equal(resfile, template) length(utilPrintOpenFiles()) == 0 end + + @test begin + filename="bad.cmap" + DEBUG && println(filename) + path = local_files(filename) + io = util_open(path, "r") + cmap = nothing + try + cmap = read_cmap(io) + finally + util_close(io) + end + @test cmap !== nothing + @test all(get_encoded_string(UInt8[0x00, 0xff], cmap).== [Char(0x0111)]) + @test all(get_encoded_string(UInt8[0x01, 0x08], cmap).== [Char(0x0110)]) + all(get_encoded_string(UInt8[0x00, 0xfb, 0x00, 0xfe], cmap) .== [Char(0x0106), Char(0x010D)]) + end + + @test begin + filename="bad1.cmap" + DEBUG && println(filename) + path = local_files(filename) + io = util_open(path, "r") + cmap = nothing + try + cmap = read_cmap(io) + finally + util_close(io) + end + @test cmap !== nothing + + @test all(get_encoded_string(UInt8[0x00, 0xff], cmap).== [Char(0x013F)]) + @test all(get_encoded_string(UInt8[0x03, 0x00], cmap).== [Char(0x0340)]) + all(get_encoded_string(UInt8[0x00, 0xfb, 0x00, 0xfe], cmap) .== [Char(0x013B), Char(0x013E)]) + end end @testset "Corrupt File" begin