diff --git a/README.md b/README.md index 8c0c82d..ad35571 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,7 @@ As can be seen above, granular APIs are provided in `PDFIO` that can be used in - Extracting document metadata information ([`pdDocGetInfo`](@ref)) - Validation of signatures in a PDF document ([`pdDocValidateSignatures`](@ref)) - Extracting fonts and font attributes ([`pdPageGetFonts`](@ref), [`pdFontIsItalic`](@ref) etc.) + - Extracting files embedded in a document ([`pdDocExtractAttachments`](@ref)), e.g. `pdDocExtractAttachments("file.pdf")` 3. Access low level PDF objects and obtain information when high level APIs do not exist. The [Architecture and Design](@ref) discusses some of these scenarios. diff --git a/docs/src/arch.md b/docs/src/arch.md index 4b5284b..0c71319 100644 --- a/docs/src/arch.md +++ b/docs/src/arch.md @@ -204,9 +204,11 @@ methods of the PD layer. #### Example -Extracting files embedded inside a PDF document is not currently available as a -PD Layer functionality. However, the same has been achieved using COS layer -functions. The code is available in the automated test cases as well. +Extracting files embedded inside a PDF document is available in the PD Layer as +[`pdDocExtractAttachments`](@ref). Before that, COS layer functions had to be +used, as in the sketch below. It is kept to illustrate the COS layer: it only +visits the page annotations and it trusts the file name stored in the PDF, so +use the PD Layer function for real work. ```julia function pdfhlp_extract_doc_attachment_files (filename, dir=tempdir()) diff --git a/docs/src/index.md b/docs/src/index.md index f100e7f..d87b8d3 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -64,6 +64,12 @@ pdDocGetPageLabel pdDocGetOutline pdDocHasSignature pdDocValidateSignatures +PDAttachment +pdDocGetAttachments +pdDocExtractAttachments +pdAttachmentGetName +pdAttachmentGetData +pdAttachmentExtract pdPageGetContents pdPageIsEmpty pdPageGetCosObject diff --git a/src/CosDoc.jl b/src/CosDoc.jl index 414b4a9..f5aec47 100644 --- a/src/CosDoc.jl +++ b/src/CosDoc.jl @@ -423,8 +423,11 @@ attach_object(doc::CosDocImpl, indstm::CosIndirectObject{CosStream})= attach_object(doc,indstm.obj) function attach_object(doc::CosDocImpl, stm::CosStream) - tmpfile = get(get(stm, cn"F")) - push!(doc.tmpfiles, String(tmpfile)) + tmpfile = String(get(get(stm, cn"F"))) + # A stream can name an external file of the PDF author's choosing in /F. Only + # the files written by the parser itself are removed when the document is closed. + startswith(abspath(tmpfile), joinpath(get_tempdir(), "")) && + push!(doc.tmpfiles, tmpfile) return nothing end diff --git a/src/PD.jl b/src/PD.jl index 619d1b9..18ece7c 100644 --- a/src/PD.jl +++ b/src/PD.jl @@ -12,5 +12,6 @@ include("PDPage.jl") include("PDFontTables.jl") include("PDOutline.jl") include("PDSignature.jl") +include("PDAttachment.jl") end diff --git a/src/PDAttachment.jl b/src/PDAttachment.jl new file mode 100644 index 0000000..26cf9b4 --- /dev/null +++ b/src/PDAttachment.jl @@ -0,0 +1,279 @@ +export PDAttachment, + pdDocGetAttachments, + pdAttachmentGetName, + pdAttachmentGetData, + pdAttachmentExtract, + pdDocExtractAttachments + +using ..Common, ..Cos + +""" +``` + PDAttachment +``` +A file embedded inside a PDF document. It's either listed in the document level +`EmbeddedFiles` name tree or attached to a page with a `FileAttachment` +annotation. + +See [`pdDocGetAttachments`](@ref). +""" +struct PDAttachment + name::String + stream::CosObject +end + +show(io::IO, att::PDAttachment) = print(io, "PDAttachment(", repr(att.name), ")") + +""" +``` + pdAttachmentGetName(att::PDAttachment) -> String +``` +Returns the file name of the attachment as recorded in the PDF document. The +name is not sanitized. It should not be used as a path without care. See +[`pdAttachmentExtract`](@ref). +""" +pdAttachmentGetName(att::PDAttachment) = att.name + +""" +``` + pdAttachmentGetData(att::PDAttachment) -> Vector{UInt8} +``` +Reads and decodes the contents of the attachment. +""" +function pdAttachmentGetData(att::PDAttachment) + bufstm = get(att.stream) + try + return read(bufstm) + finally + util_close(bufstm) + end +end + +# Attachment file names are not trusted. Only a plain file name is retained. +function sanitize_filename(name::AbstractString) + name = String(last(split(replace(name, '\\' => '/'), '/'))) + name = replace(name, r"[\x00-\x1f\x7f-\x9f<>:\"|?*]" => "_") + name = String(rstrip(strip(name), ['.', ' '])) + # Windows device names are unusable as file names, even with an extension. + occursin(r"^(CON|PRN|AUX|NUL|COM[0-9]|LPT[0-9])(\..*)?$"i, name) && + (name = "_" * name) + return name in ("", ".", "..") ? "attachment" : name +end + +# Creates a new file in `dir`. `O_EXCL` makes the creation atomic, so an existing +# file or a symbolic link is never opened. A numeric suffix is tried instead. +function create_new_file(dir::AbstractString, name::String) + base, ext = splitext(name) + for i = 0:typemax(Int16) + path = joinpath(dir, i == 0 ? name : string(base, " (", i, ")", ext)) + try + flags = Base.Filesystem.JL_O_WRONLY | Base.Filesystem.JL_O_CREAT | + Base.Filesystem.JL_O_EXCL + # 0o666 is subject to the umask of the user, like a regular `write`. + return path, Base.Filesystem.open(path, flags, 0o666) + catch e + (e isa Base.IOError && e.code == Base.UV_EEXIST) || rethrow() + end + end + error("Could not find an unused file name for $name in $dir") +end + +""" +``` + pdAttachmentExtract(att::PDAttachment, dir::AbstractString=".") -> String +``` +Writes the attachment into the directory `dir` and returns the path of the file +written. Only the file name part of the attachment name is used. Existing files +are never overwritten. A numeric suffix is added to the file name instead. +""" +function pdAttachmentExtract(att::PDAttachment, dir::AbstractString=".") + data = pdAttachmentGetData(att) # No file is created if decoding fails. + mkpath(dir) + path, io = create_new_file(dir, sanitize_filename(att.name)) + try + write(io, data) + close(io) # Can fail while flushing, hence inside the `try`. + catch + try close(io) catch end # The original error is the one to report. + rm(path; force=true) # Do not leave a partial file behind. + rethrow() + end + return path +end + +# Text strings are UTF-16BE or UTF-8 with a byte order mark, else PDFDocEncoding. +function pdf_text(str::CosString, fallback::String="attachment") + b = Vector{UInt8}(str) + if length(b) >= 2 && b[1] == 0xfe && b[2] == 0xff + # An odd number of bytes is not valid UTF-16: do not truncate the name. + isodd(length(b)) && return fallback + u16 = UInt16[(UInt16(b[i]) << 8) | b[i+1] for i = 3:2:length(b)-1] + s = transcode(String, u16) + # An unpaired surrogate transcodes without error but leaves invalid + # UTF-8 bytes behind (e.g. a lone high surrogate). + return isvalid(s) ? s : fallback + elseif length(b) >= 3 && b[1:3] == UInt8[0xef, 0xbb, 0xbf] + return isvalid(String, b[4:end]) ? String(b[4:end]) : fallback + end + return String(CDTextString(PDFEncodingToUnicode(b))) +end + +is_internal_stream(stm::CosStream) = stm.isInternal +is_internal_stream(stm::CosIndirectObject{CosStream}) = stm.obj.isInternal + +# The file specification dictionary is resolved to the embedded file stream. +function attachment_from_filespec(cosdoc::CosDoc, fs::CosObject, + fallback::String="attachment") + fsdict = cosDocGetObject(cosdoc, fs) + fsdict isa IDD{CosDict} || return nothing + ef = cosDocGetObject(cosdoc, fsdict, cn"EF") + ef isa IDD{CosDict} || return nothing + stm = CosNull + for key in (cn"F", cn"UF", cn"Unix", cn"Mac", cn"DOS") # Legacy keys are deprecated. + stm = cosDocGetObject(cosdoc, ef, key) + stm === CosNull || break + end + stm isa IDD{CosStream} || return nothing + # A stream whose /F was supplied by the PDF itself (as opposed to one the + # parser wrote out internally) refers to an arbitrary local file path. + # Reading it would let a crafted PDF exfiltrate files readable by this + # process, so such streams are not treated as attachments. + is_internal_stream(stm) || return nothing + name = fallback + for key in (cn"UF", cn"F") + nobj = cosDocGetObject(cosdoc, fsdict, key) + nobj isa CosIndirectObject && (nobj = nobj.obj) + if nobj isa CosString + name = pdf_text(nobj, fallback) + break + end + end + return PDAttachment(name, stm) +end + +# Walks a name tree calling `fn(key, value)` on every entry. `/Names` and `/Kids` +# may be indirect objects, hence they are resolved through the document. +function collect_nametree!(fn::Function, cosdoc::CosDoc, node::IDD{CosDict}, + visited::Set{CosIndirectObjectRef}) + names = cosDocGetObject(cosdoc, node, cn"Names") + if names isa IDD{CosArray} + v = get(names) + for i = 1:2:length(v)-1 + key = cosDocGetObject(cosdoc, v[i]) + fn(key isa CosString ? pdf_text(key) : "attachment", v[i+1]) + end + end + kids = cosDocGetObject(cosdoc, node, cn"Kids") + kids isa IDD{CosArray} || return + for kid in get(kids) + if kid isa CosIndirectObjectRef + kid in visited && continue + push!(visited, kid) + end + kidobj = cosDocGetObject(cosdoc, kid) + kidobj isa IDD{CosDict} && collect_nametree!(fn, cosdoc, kidobj, visited) + end +end + +""" +``` + pdDocGetAttachments(doc::PDDoc) -> Vector{PDAttachment} +``` +Returns all the files embedded in the document. The files listed in the +`EmbeddedFiles` name tree are followed by the ones attached to the pages with +`FileAttachment` annotations. An embedded file referred in both the places is +returned once. Encrypted documents are not supported. + +# Example +``` +julia> pdDocGetAttachments(doc) +2-element Vector{PDAttachment}: + PDAttachment("hello.txt") + PDAttachment("data.bin") +``` +""" +function pdDocGetAttachments(doc::PDDoc) + cosdoc = pdDocGetCosDoc(doc) + atts, seen = PDAttachment[], Set{Any}() + function add!(fs, fallback="attachment") + att = attachment_from_filespec(cosdoc, fs, fallback) + att === nothing && return + # Same embedded file stream can be referenced from multiple places. + id = att.stream isa CosIndirectObject ? + (att.stream.num, att.stream.gen) : objectid(att.stream) + id in seen && return + push!(seen, id) + push!(atts, att) + end + + names = pdDocGetNamesDict(doc) + if names isa IDD{CosDict} + ef = cosDocGetObject(cosdoc, names, cn"EmbeddedFiles") + if ef isa IDD{CosDict} + visited = Set{CosIndirectObjectRef}() + collect_nametree!(cosdoc, ef, visited) do key, fs + add!(fs, key) + end + end + end + + for i = 1:pdDocGetPageCount(doc) + cospage = pdPageGetCosObject(pdDocGetPage(doc, i)) + annots = cosDocGetObject(cosdoc, cospage, cn"Annots") + annots isa IDD{CosArray} || continue + for annot in get(annots) + adict = cosDocGetObject(cosdoc, annot) + adict isa IDD{CosDict} || continue + cosDocGetObject(cosdoc, adict, cn"Subtype") === cn"FileAttachment" || + continue + add!(cosDocGetObject(cosdoc, adict, cn"FS")) + end + end + return atts +end + +""" +``` + pdDocExtractAttachments(doc::PDDoc, dir::AbstractString=".") -> Vector{String} +``` +Extracts all the files embedded in the document into the directory `dir` +(default is the current directory), creating it if required. Returns the paths +of the files written. See [`pdAttachmentExtract`](@ref). + +# Example +``` +julia> doc = pdDocOpen("invoice.pdf"); + +julia> pdDocExtractAttachments(doc) +1-element Vector{String}: + "./invoice.xml" +``` +""" +function pdDocExtractAttachments(doc::PDDoc, dir::AbstractString=".") + return [pdAttachmentExtract(att, dir) for att in pdDocGetAttachments(doc)] +end + +""" +``` + pdDocExtractAttachments(filepath::AbstractString, dir::AbstractString=".") -> Vector{String} +``` +Convenience method that opens the PDF document at `filepath`, extracts every +file embedded in it into the directory `dir` (default is the current +directory), and closes the document. Returns the paths of the files written. +See [`pdDocExtractAttachments(::PDDoc, ::AbstractString)`](@ref). + +# Example +``` +julia> pdDocExtractAttachments("invoice.pdf") +1-element Vector{String}: + "./invoice.xml" +``` +""" +function pdDocExtractAttachments(filepath::AbstractString, dir::AbstractString=".") + doc = pdDocOpen(filepath) + try + return pdDocExtractAttachments(doc, dir) + finally + pdDocClose(doc) + end +end diff --git a/src/PDFIO.jl b/src/PDFIO.jl index d45d636..7b701b7 100644 --- a/src/PDFIO.jl +++ b/src/PDFIO.jl @@ -25,6 +25,8 @@ export PDDoc, pdDocGetOutline, pdDocHasSignature, pdDocValidateSignatures, + pdDocGetAttachments, + pdDocExtractAttachments, PDPage, pdPageGetContents, pdPageIsEmpty, @@ -52,7 +54,11 @@ export PDDoc, PDOutline, PDDestination, PDOutlineItem, - pdOutlineItemGetAttr + pdOutlineItemGetAttr, + PDAttachment, + pdAttachmentGetName, + pdAttachmentGetData, + pdAttachmentExtract using .Cos export CosDoc, diff --git a/test/files/attachments.pdf b/test/files/attachments.pdf new file mode 100644 index 0000000..43192b6 Binary files /dev/null and b/test/files/attachments.pdf differ diff --git a/test/files/attachments_edge.pdf b/test/files/attachments_edge.pdf new file mode 100644 index 0000000..31b7b2b Binary files /dev/null and b/test/files/attachments_edge.pdf differ diff --git a/test/runtests.jl b/test/runtests.jl index b93cbfb..1146c38 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,7 +9,8 @@ using Downloads # Internal methods for testing only using PDFIO.Cos: parse_indirect_ref, decode_ascii85, CosXString, parse_value -using PDFIO.PD: openssl_error, read_cmap, get_encoded_string +using PDFIO.PD: openssl_error, read_cmap, get_encoded_string, is_internal_stream, + pdf_text using PDFIO.Common: read_pkcs12 include("debugIO.jl") @@ -818,6 +819,147 @@ local_files(filename, filesdir="files") = joinpath(@__DIR__, pdftest_dir, filesd @test length(utilPrintOpenFiles()) == 0 end + @testset "Attachment extraction" begin + # test/files/attachments.pdf has 2 files in the EmbeddedFiles name tree + # and 1 in a FileAttachment annotation. + src = joinpath(@__DIR__, "files", "attachments.pdf") + expected = Dict("hello.txt" => Vector{UInt8}("Hello from PDFIO attachments\n"), + "data.bin" => repeat(UInt8.(0:255), 4), + "note.csv" => Vector{UInt8}("a,b\n1,2\n")) + doc = pdDocOpen(src) + atts = pdDocGetAttachments(doc) + @test sort(pdAttachmentGetName.(atts)) == sort(collect(keys(expected))) + for att in atts + @test pdAttachmentGetData(att) == expected[pdAttachmentGetName(att)] + end + mktempdir() do dir + outdir = joinpath(dir, "out") + paths = pdDocExtractAttachments(doc, outdir) + @test sort(basename.(paths)) == sort(collect(keys(expected))) + for (name, data) in expected + @test read(joinpath(outdir, name)) == data + end + # Existing files are never overwritten. + again = pdDocExtractAttachments(doc, outdir) + @test all(p -> occursin(" (1)", basename(p)), again) + @test read(joinpath(outdir, "hello.txt")) == expected["hello.txt"] + @test read(joinpath(outdir, "hello (1).txt")) == expected["hello.txt"] + end + pdDocClose(doc) + @test length(utilPrintOpenFiles()) == 0 + + # Convenience method: open, extract, and close in one call. + mktempdir() do dir + paths = pdDocExtractAttachments(src, dir) + @test sort(basename.(paths)) == sort(collect(keys(expected))) + for (name, data) in expected + @test read(joinpath(dir, name)) == data + end + end + @test length(utilPrintOpenFiles()) == 0 + # No file handle leaks if opening the document throws... + @test_throws SystemError pdDocExtractAttachments("/nonexistent/file.pdf") + @test length(utilPrintOpenFiles()) == 0 + # ...nor if extraction itself throws after the document is opened. + mktempdir() do dir + blocked = joinpath(dir, "not_a_dir") + write(blocked, "x") # A regular file where a directory is needed. + @test_throws Base.IOError pdDocExtractAttachments(src, blocked) + end + @test length(utilPrintOpenFiles()) == 0 + + # Names stored in a document are not trusted as paths. + sanitize = PDFIO.PD.sanitize_filename + @test sanitize("../../etc/passwd") == "passwd" + @test sanitize("C:\\dir\\a.txt") == "a.txt" + @test sanitize("..") == "attachment" + @test sanitize("") == "attachment" + @test sanitize("a:b?.txt") == "a_b_.txt" + + # A stream whose /F was supplied by the PDF itself (a path outside + # the parser's own tempdir) must never be treated as an attachment. + @test is_internal_stream(CosStream(CosDict())) == true + @test is_internal_stream(CosStream(CosDict(), false)) == false + + # An unpaired UTF-16 surrogate transcodes without error but leaves + # invalid UTF-8 behind; a UTF-8-BOM string can be malformed the same + # way. Both must fall back rather than propagate an invalid String. + lone_surrogate = CosLiteralString(UInt8[0xfe, 0xff, 0xd8, 0x00]) + @test pdf_text(lone_surrogate, "fb") == "fb" + bad_utf8_bom = CosLiteralString(UInt8[0xef, 0xbb, 0xbf, 0xff]) + @test pdf_text(bad_utf8_bom, "fb") == "fb" + @test pdf_text(CosLiteralString(UInt8[0xef, 0xbb, 0xbf, 0x41]), "fb") == "A" + + @test sanitize("CON.txt") == "_CON.txt" + @test sanitize("nul") == "_nul" + @test sanitize("console.txt") == "console.txt" + @test sanitize("name. .") == "name" + @test sanitize("a\u009bb\e[31m") == "a_b_[31m" # C0 and C1 controls + + # UTF-16BE text with an odd number of bytes is malformed: use the fallback. + @test pdf_text(CosLiteralString(UInt8[0xfe, 0xff, 0x00, 0x41])) == "A" + @test pdf_text(CosLiteralString(UInt8[0xfe, 0xff, 0x00, 0x41, 0x00]), "fb") == "fb" + + # Indirect /Names and /Kids arrays, UTF-16BE literal name (/UF is + # preferred over /F), one byte hex name <41>, reserved and traversal names. + doc = pdDocOpen(joinpath(@__DIR__, "files", "attachments_edge.pdf")) + atts = pdDocGetAttachments(doc) + @test pdAttachmentGetName.(atts) == + ["日本語.txt", "A", "CON.txt", "../evil.txt"] + mktempdir() do dir + paths = pdDocExtractAttachments(doc, dir) + @test basename.(paths) == ["日本語.txt", "A", "_CON.txt", "evil.txt"] + @test read.(paths, String) == ["unicode", "short", "reserved", "traversal"] + @test readdir(dir) |> sort == sort(basename.(paths)) # Nothing escaped + end + mktempdir() do dir + # An existing symbolic link is never followed. + target = joinpath(dir, "target") + if (try symlink(target, joinpath(dir, "A")); true catch; false end) + pdAttachmentExtract(atts[2], dir) + @test !ispath(target) + @test read(joinpath(dir, "A (1)"), String) == "short" + end + end + pdDocClose(doc) + @test length(utilPrintOpenFiles()) == 0 + + # A stream naming an external file in /F is not an attachment, and the + # file must survive closing the document. The legacy /EF /Unix key is read. + mktempdir() do dir + victim = joinpath(dir, "victim.txt") + write(victim, "keep") + objs = ["<< /Type /Catalog /Pages 2 0 R /Names << /EmbeddedFiles " * + "<< /Names [(a) 4 0 R (b) 6 0 R] >> >> >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /Length 0 /F ($(replace(victim, '\\' => "\\\\"))) >>\nstream", + "<< /Type /Filespec /EF << /F 3 0 R >> /F (ext.txt) >>", + "<< /Length 6 >>\nstream\nlegacy\nendstream", + "<< /Type /Filespec /EF << /Unix 5 0 R >> /F (old.txt) >>"] + io = IOBuffer() + write(io, "%PDF-1.7\n%", " "^256, "\n") # The trailer search needs 256 bytes. + offsets = Int[] + for (i, obj) in enumerate(objs) + push!(offsets, position(io)) + write(io, "$i 0 obj\n$obj\nendobj\n") + end + xref = position(io) + write(io, "xref\n0 $(length(objs) + 1)\n0000000000 65535 f \n") + foreach(o -> write(io, lpad(o, 10, '0'), " 00000 n \n"), offsets) + write(io, "trailer\n<< /Size $(length(objs) + 1) /Root 1 0 R >>\n", + "startxref\n$xref\n%%EOF\n") + crafted = joinpath(dir, "crafted.pdf") + write(crafted, take!(io)) + doc = pdDocOpen(crafted) + atts = pdDocGetAttachments(doc) + @test pdAttachmentGetName.(atts) == ["old.txt"] + @test pdAttachmentGetData(atts[1]) == Vector{UInt8}("legacy") + pdDocClose(doc) + @test read(victim, String) == "keep" + end + @test length(utilPrintOpenFiles()) == 0 + end + files=readdir(get_tempdir()) @assert length(files) == 0 end