Hello
world
' + self.assertEqual(anydoc.format_from_extension('html'), 'html') + self.assertEqual(anydoc.format_from_bytes(data), 'html') + self.assertEqual(anydoc.to_markdown_bytes(data), '# Hello\n\n**world**\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/src/formats/detect.rs b/src/formats/detect.rs index 727a69f6..522e513b 100644 --- a/src/formats/detect.rs +++ b/src/formats/detect.rs @@ -1,7 +1,9 @@ //! Content-based format detection. //! -//! Identifies the format from what each specification designates as the -//! container's identity, never from heuristics over document content: +//! Identifies the format primarily from what each specification designates as +//! the container's identity. Standalone HTML is the one intentional content +//! heuristic: after an optional BOM/leading whitespace, a leading HTML5 +//! `` or `` marker identifies `Format::Html`. //! //! - PDF: the `%PDF-` header (ISO 32000; implementations accept leading //! junk, bounded here at 1024 bytes). @@ -41,12 +43,117 @@ pub(crate) fn from_bytes(bytes: &[u8]) -> Optionwhen a block-level start tag + // arrives. Model the case where the
is the innermost open element; + // deeper arrangements stay over-counted, keeping the preflight + // fail-closed. (hr also closes
in HTML5 but is void, so it never + // reaches this hook; the leftover
only over-counts.) + if is_paragraph_closing_element(name) + && open.last().is_some_and(|candidate| candidate.as_ref() == "p") + { + open.pop(); + } + + if is_heading_element(name) + && open.last().is_some_and(|candidate| is_heading_element(candidate.as_ref())) + { + open.pop(); + } + + let implied = match name { + "li" => &["li"][..], + "p" => &["p"][..], + "dt" | "dd" => &["dt", "dd"][..], + "rt" | "rp" => &["rt", "rp"][..], + "option" => &["option"][..], + "optgroup" => &["option", "optgroup"][..], + "tr" => &["tr"][..], + "td" | "th" => &["td", "th"][..], + _ => return, + }; + if let Some(position) = open.iter().rposition(|candidate| implied.contains(&candidate.as_ref())) + { + open.truncate(position); + } +} + +fn is_heading_element(name: &str) -> bool { + matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") +} + +/// Block-level start tags that make HTML5 implicitly close an open
+/// (html5ever's in-body close_p_element_in_button_scope arms; table closes +///
only outside quirks mode, and the preflight assumes standards mode).
+fn is_paragraph_closing_element(name: &str) -> bool {
+ matches!(
+ name,
+ "address"
+ | "article"
+ | "aside"
+ | "blockquote"
+ | "center"
+ | "details"
+ | "dialog"
+ | "dir"
+ | "div"
+ | "dl"
+ | "fieldset"
+ | "figcaption"
+ | "figure"
+ | "footer"
+ | "form"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ | "header"
+ | "hgroup"
+ | "hr"
+ | "listing"
+ | "main"
+ | "menu"
+ | "nav"
+ | "ol"
+ | "p"
+ | "plaintext"
+ | "pre"
+ | "search"
+ | "section"
+ | "summary"
+ | "table"
+ | "ul"
+ )
+}
+
+fn html5_self_closing_is_honored(open: &[LocalName], name: &str) -> bool {
+ if matches!(name, "svg" | "math") {
+ return true;
+ }
+
+ let mut in_foreign_content = false;
+ for candidate in open.iter().rev() {
+ match candidate.as_ref() {
+ "foreignobject" | "desc" | "title" | "mi" | "mo" | "mn" | "ms" | "mtext"
+ | "annotation-xml" => return false,
+ "svg" | "math" => {
+ in_foreign_content = true;
+ break;
+ }
+ _ => {}
+ }
+ }
+
+ in_foreign_content && !is_foreign_content_html_breakout(name)
+}
+
+fn is_foreign_content_html_breakout(name: &str) -> bool {
+ matches!(
+ name,
+ "b" | "big"
+ | "blockquote"
+ | "body"
+ | "br"
+ | "center"
+ | "code"
+ | "dd"
+ | "div"
+ | "dl"
+ | "dt"
+ | "em"
+ | "embed"
+ | "font"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ | "head"
+ | "hr"
+ | "i"
+ | "img"
+ | "li"
+ | "listing"
+ | "menu"
+ | "meta"
+ | "nobr"
+ | "ol"
+ | "p"
+ | "pre"
+ | "ruby"
+ | "s"
+ | "small"
+ | "span"
+ | "strike"
+ | "strong"
+ | "sub"
+ | "sup"
+ | "table"
+ | "tt"
+ | "u"
+ | "ul"
+ | "var"
+ )
+}
+
+fn is_void_html_element(name: &str) -> bool {
+ matches!(
+ name,
+ "area"
+ | "base"
+ | "br"
+ | "col"
+ | "embed"
+ | "hr"
+ | "img"
+ | "input"
+ | "link"
+ | "meta"
+ | "param"
+ | "source"
+ | "track"
+ | "wbr"
+ )
+}
+
+fn preflight_html_complexity(text: &str) -> Result<(), ConvertError> {
+ const CHUNK_BYTES: usize = 64 * 1024;
+ let tokenizer = Tokenizer::new(HtmlComplexitySink::default(), Default::default());
+ let input = BufferQueue::default();
+ let mut offset = 0usize;
+
+ while offset < text.len() {
+ let mut end = offset.saturating_add(CHUNK_BYTES).min(text.len());
+ while end > offset && !text.is_char_boundary(end) {
+ end -= 1;
+ }
+ input.push_back(StrTendril::from(&text[offset..end]));
+ let _ = tokenizer.feed(&input);
+ check_preflight_limits(&tokenizer.sink)?;
+ offset = end;
+ }
+
+ tokenizer.end();
+ check_preflight_limits(&tokenizer.sink)
+}
+
+fn check_preflight_limits(sink: &HtmlComplexitySink) -> Result<(), ConvertError> {
+ if sink.node_limit_exceeded.get() {
+ return Err(ConvertError::ResourceLimit {
+ limit: "max_xml_nodes",
+ detail: format!("HTML token stream has more than {} nodes", limits::MAX_XML_NODES),
+ });
+ }
+ if sink.depth_limit_exceeded.get() {
+ return Err(ConvertError::ResourceLimit {
+ limit: "max_xml_depth",
+ detail: format!(
+ "HTML source nesting depth exceeds {} before DOM construction",
+ limits::MAX_XML_DEPTH
+ ),
+ });
+ }
+ Ok(())
+}
+
+fn adapt_element(
+ source: ElementRef<'_>,
+ depth: usize,
+ node_count: &mut usize,
+) -> Result utf-8 utf-8 Plain paragraph with bold, italic, and example. café Ł music 𝄞 family 👨👩👧. Styled bold paragraph. drop me One-one One-two One-two-a
+ roman One-two-b
+ roman Deep bullet One-three Interruption
+paragraph. One-four
+ continues the count Two-one
+ independent counter Ten-start
+ via override Letter-A via
+ numStyleLink Level
+ replaced and start overridden together Suppressed numbering
+paragraph Style-numbered
+ paragraph pStyle-bound
+ level one pStyle-bound level
+two pStyle-bound
+ level one again Inherited
+subheading Bold here, and
+style-false keeps it bold toggled
+off by a true in the character style double toggle
+cancels to plain direct
+off wins absolutely direct on makes
+this italic
+Plain paragraph with bold, italic, and Style-bold
+paragraph with a NotBold-styled
+span inside.
+ First numbered
+ Second numbered
+ Alpha sub one
+ Alpha sub two
+ Roman sub sub
+ Third numbered
+Interrupting paragraph between lists.
+ Fourth, continuing the count
+ Roman starting at four
+ Roman five
+ Bullet one
+ Bullet two
+ Nested bullet
+ Wide head
+ End
+ Tall
+ B2
+ C2
+ B3
+ C3
+Music clef 𝄞 appears before this footnote1
+reference.
+An endnote follows herei.
+Persian with ZWNJ: میخواهم.
+Family emoji: 👨👩👧.
+Markdown specials: *stars* _under_ [bracket] `tick` #hash 1. dotted |
+pipe.
+External link to example.
+Relative link to a
+sibling file.
+This plain paragraph carries a bookmark.
+Jump to the bookmarked
+paragraph.
+Inline image:
+
+Text box: after the box.
+ Inside the text box.
+Value below one millionth: 0.0000004 should survive. 1Footnote
+ after an astral character. iEndnote
+ body text. Hello café café café first second"#;
+ let markdown = to_markdown_bytes(html, None).unwrap();
+ assert_eq!(markdown, "# Hello\n\nfirst\n\n**second**\n");
+}
+
+#[test]
+fn html5_table_inserts_implicit_structure() {
+ let html = br#" drop me keep me caf".to_vec();
+ html.push(0xE9);
+ html.extend_from_slice(b" before after "#.to_vec();
+ html.push(0x80);
+ html.extend_from_slice(b" paragraph {i}"));
+ }
+ let markdown = to_markdown_bytes(html.as_bytes(), Some(Format::Html)).unwrap();
+ assert!(markdown.contains("paragraph 299"));
+}
+
+#[test]
+fn meta_charset_after_first_kib_is_still_honored() {
+ let mut html = b" ");
+ html.push(0xA3);
+ html.extend_from_slice(b" ok when the text text = None;
let mut last_number = 0i64;
- for (li, &number) in items.iter().zip(&numbers) {
- let item = ListItem { blocks: self.sub_blocks(li, delta)?, marker_label: None };
+ let mut item_index = 0usize;
+
+ for child in children {
+ if child.local != "li" {
+ let child_blocks = self.sub_element_blocks(child, delta)?;
+ if child_blocks.is_empty() {
+ continue;
+ }
+ if let Some(list) = current.take() {
+ out.push(Block::List(list));
+ }
+ out.extend(child_blocks);
+ continue;
+ }
+
+ let number = numbers[item_index];
+ item_index += 1;
+ let item = ListItem {
+ blocks: self.sub_blocks(child, delta)?,
+ marker_label: explicit_markers.then(|| format!("{number}.")),
+ };
+
+ if explicit_markers {
+ if current.is_none() {
+ current = Some(List { marker, start: 1, items: Vec::new() });
+ }
+ current.as_mut().unwrap().items.push(item);
+ continue;
+ }
+
let contiguous = current.is_some() && last_number.checked_add(1) == Some(number);
if !contiguous {
if let Some(list) = current.take() {
diff --git a/tests/fixtures/html/controlled-document.html b/tests/fixtures/html/controlled-document.html
new file mode 100644
index 00000000..88fa4c65
--- /dev/null
+++ b/tests/fixtures/html/controlled-document.html
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
Fixture Document
+ Unicode
+ Lists
+
+
+
+
diff --git a/tests/fixtures/html/controlled-merged-table.html b/tests/fixtures/html/controlled-merged-table.html
new file mode 100644
index 00000000..15ff1adb
--- /dev/null
+++ b/tests/fixtures/html/controlled-merged-table.html
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/fixtures/html/libreoffice-docx-numbering.html b/tests/fixtures/html/libreoffice-docx-numbering.html
new file mode 100644
index 00000000..79242746
--- /dev/null
+++ b/tests/fixtures/html/libreoffice-docx-numbering.html
@@ -0,0 +1,99 @@
+
+
+
+
+ Merged across padded tall b2 3.5 b3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Fixture Document
struck
+runs.
+
+
+
+Lists
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Table
+
+
+
+
+
+
+
+ Notes and special text
+
+
+
+Links and anchors
+
+
+
+Objects
+
+done.
+
+
+
+
+Quote and code
+
+
+
+
+
diff --git a/tests/html.rs b/tests/html.rs
new file mode 100644
index 00000000..caf7744d
--- /dev/null
+++ b/tests/html.rs
@@ -0,0 +1,376 @@
+use anydoc::model::{Block, ImageSource, Inline};
+use anydoc::{ConvertError, Format, to_document, to_markdown_bytes};
+
+#[test]
+fn html_extensions_are_named() {
+ assert_eq!(Format::from_extension("html"), Some(Format::Html));
+ assert_eq!(Format::from_extension("HTM"), Some(Format::Html));
+}
+
+#[test]
+fn html_doctype_is_detected_from_content() {
+ let html = b"\xEF\xBB\xBF
+
+ Merged across
+ padded
+
+
+ tall
+ b2
+ 3.5
+
+
+b3
+ Hello
"#;
+ let markdown = to_markdown_bytes(html, None).unwrap();
+ assert_eq!(markdown, "| A | B |\n| --- | --- |\n| 1 | 2 |\n");
+}
+
+#[test]
+fn style_blocks_feed_the_existing_semantic_css_subset() {
+ let html = br#"A B 1 2 ");
+ for i in 0..300 {
+ html.push_str(&format!("
");
+ let markdown = to_markdown_bytes(html.as_bytes(), Some(Format::Html)).unwrap();
+ assert!(markdown.contains("item 299"));
+}
+
+#[test]
+fn optional_p_end_tags_do_not_count_as_nested_depth() {
+ let mut html = String::from("");
+ for i in 0..300 {
+ html.push_str(&format!("
heading {i}"));
+ }
+
+ let markdown = to_markdown_bytes(html.as_bytes(), Some(Format::Html)).unwrap();
+ assert!(markdown.contains("# heading 299"));
+}
+
+#[test]
+fn alternating_headings_are_implicitly_closed_before_preflight_depth_counting() {
+ const HEADINGS: [&str; 6] = ["h1", "h2", "h3", "h4", "h5", "h6"];
+ let mut html = String::from("");
+ for i in 0..300 {
+ let heading = HEADINGS[i % HEADINGS.len()];
+ html.push_str(&format!("<{heading}>heading {i}"));
+ }
+
+ let markdown = to_markdown_bytes(html.as_bytes(), Some(Format::Html)).unwrap();
+ assert!(markdown.contains("heading 299"));
+}
+
+#[test]
+fn foreign_self_closing_svg_elements_do_not_accumulate_html_depth() {
+ let mut html = String::from("
"#;
+ let markdown = to_markdown_bytes(html, Some(Format::Html)).unwrap();
+ assert!(markdown.contains("Nested heading"), "{markdown:?}");
+ assert!(markdown.contains("| A |"), "{markdown:?}");
+}
+
+#[test]
+fn repeated_unclosed_anchors_do_not_trigger_depth_limit() {
+ let mut html = String::from("");
+ for index in 0..300 {
+ html.push_str(&format!("link"));
+ }
+ let result = to_markdown_bytes(html.as_bytes(), Some(Format::Html));
+ assert!(
+ !matches!(result, Err(ConvertError::ResourceLimit { limit: "max_xml_depth", .. })),
+ "HTML5 repairs repeated anchors; preflight must not reject them as excessive depth"
+ );
+}
+
+#[test]
+fn intervening_blocks_between_unclosed_anchors_still_hit_preflight_depth_limit() {
+ let mut html = String::from("");
+ for index in 0..300 {
+ html.push_str(&format!("
+
Nested heading
+ A 
+
"#;
+
+ let markdown = to_markdown_bytes(html, Some(Format::Html)).unwrap();
+ assert_eq!(markdown, "- one\n- two\n");
+}
+
+#[test]
+fn ordered_list_ignores_non_rendering_children_without_splitting() {
+ let html = br#"
+
+
"#;
+
+ let markdown = to_markdown_bytes(html, Some(Format::Html)).unwrap();
+ assert_eq!(markdown, "1. one\n2. two\n");
+}
diff --git a/tests/snapshots.rs b/tests/snapshots.rs
index 25f997c2..9b77eee6 100644
--- a/tests/snapshots.rs
+++ b/tests/snapshots.rs
@@ -85,6 +85,7 @@ fn fixtures_detect_from_content() {
("doc", Some(Format::Doc)),
("docx", Some(Format::Docx)),
("epub", Some(Format::Epub)),
+ ("html", Some(Format::Html)),
("odp", Some(Format::Odp)),
("ods", Some(Format::Ods)),
("odt", Some(Format::Odt)),
diff --git a/tests/snapshots/snapshots__html__controlled-document.html.snap b/tests/snapshots/snapshots__html__controlled-document.html.snap
new file mode 100644
index 00000000..0e3964eb
--- /dev/null
+++ b/tests/snapshots/snapshots__html__controlled-document.html.snap
@@ -0,0 +1,13 @@
+---
+source: tests/snapshots.rs
+expression: output
+---
+# Fixture Document
+
+Plain paragraph with **bold**, *italic*, and [example](https://example.com/page).
+
+## Unicode
+
+café Ł music 𝄞 family 👨👩👧.
+
+**Styled bold paragraph.**
diff --git a/tests/snapshots/snapshots__html__controlled-lists.html.snap b/tests/snapshots/snapshots__html__controlled-lists.html.snap
new file mode 100644
index 00000000..f52b3598
--- /dev/null
+++ b/tests/snapshots/snapshots__html__controlled-lists.html.snap
@@ -0,0 +1,12 @@
+---
+source: tests/snapshots.rs
+expression: output
+---
+## Lists
+
+3. Third
+
+4. Fourth
+
+ - Nested A
+ - Nested B
diff --git a/tests/snapshots/snapshots__html__controlled-merged-table.html.snap b/tests/snapshots/snapshots__html__controlled-merged-table.html.snap
new file mode 100644
index 00000000..d32b3294
--- /dev/null
+++ b/tests/snapshots/snapshots__html__controlled-merged-table.html.snap
@@ -0,0 +1,9 @@
+---
+source: tests/snapshots.rs
+expression: output
+---
+| | | |
+| --- | --- | --- |
+| Merged across | | padded |
+| tall | b2 | 3.5 |
+| | b3 | |
diff --git a/tests/snapshots/snapshots__html__libreoffice-docx-numbering.html.snap b/tests/snapshots/snapshots__html__libreoffice-docx-numbering.html.snap
new file mode 100644
index 00000000..ebd4ae55
--- /dev/null
+++ b/tests/snapshots/snapshots__html__libreoffice-docx-numbering.html.snap
@@ -0,0 +1,49 @@
+---
+source: tests/snapshots.rs
+expression: output
+---
+1. One-one
+
+2. One-two
+
+ - i. One-two-a roman
+
+ - ii. One-two-b roman
+
+ - Deep bullet
+
+3. One-three
+
+Interruption paragraph.
+
+4. One-four continues the count
+
+1. Two-one independent counter
+
+10. Ten-start via override
+
+- A. Letter-A via numStyleLink
+
+- G. Level replaced and start overridden together
+
+Suppressed numbering paragraph
+
+8. Style-numbered paragraph
+
+1. pStyle-bound level one
+
+pStyle-bound level two
+
+2. pStyle-bound level one again
+
+**Inherited subheading**
+
+**Bold here,** and style-false keeps it bold
+
+toggled off by a true in the character style
+
+**double toggle cancels to plain**
+
+direct off wins absolutely
+
+*direct on makes this italic*
diff --git a/tests/snapshots/snapshots__html__libreoffice-docx-text.html.snap b/tests/snapshots/snapshots__html__libreoffice-docx-text.html.snap
new file mode 100644
index 00000000..566b5585
--- /dev/null
+++ b/tests/snapshots/snapshots__html__libreoffice-docx-text.html.snap
@@ -0,0 +1,83 @@
+---
+source: tests/snapshots.rs
+expression: output
+---
+1. # Fixture Document
+
+Plain paragraph with **bold**, *italic*, and ~~struck~~ runs.
+
+**Style-bold paragraph with a** NotBold-styled span **inside.**
+
+1. ## Lists
+
+1. First numbered
+
+2. Second numbered
+
+ - a. Alpha sub one
+
+ - b. Alpha sub two
+
+ - i. Roman sub sub
+
+3. Third numbered
+
+Interrupting paragraph between lists.
+
+4. Fourth, continuing the count
+
+- IV. Roman starting at four
+
+- I. Roman five
+
+- Bullet one
+
+- Bullet two
+
+ - Nested bullet
+
+2. ## Table
+
+| | | | |
+| --- | --- | --- | --- |
+| Wide head | | End | |
+| Tall | B2 | | C2 |
+| | B3 | | C3 |
+
+3. ## Notes and special text
+
+Music clef 𝄞 appears before this footnote[1](#sdfootnote1sym) reference.
+
+An endnote follows here[i](#sdendnote1sym).
+
+Persian with ZWNJ: میخواهم. Family emoji: 👨👩👧.
+
+Markdown specials: \*stars* \_under_ \[bracket] \`tick` #hash 1. dotted | pipe.
+
+4. ## Links and anchors
+
+External link to [example](https://example.com/page).
+
+Relative link to [a sibling file](../../tests/fixture-src/sibling.odt).
+
+This plain paragraph carries a bookmark.
+
+Jump to [the bookmarked paragraph](#plainmark).
+
+5. ## Objects
+
+Inline image:  done.
+
+Text box: after the box.
+
+Inside the text box.
+
+
+
+6. ## Quote and code
+
+Value below one millionth: 0.0000004 should survive.
+
+[1](#sdfootnote1anc)Footnote after an astral character.
+
+[i](#sdendnote1anc)Endnote body text.
diff --git a/tests/snapshots/snapshots__html__libreoffice-xlsx-merged.html.snap b/tests/snapshots/snapshots__html__libreoffice-xlsx-merged.html.snap
new file mode 100644
index 00000000..d32b3294
--- /dev/null
+++ b/tests/snapshots/snapshots__html__libreoffice-xlsx-merged.html.snap
@@ -0,0 +1,9 @@
+---
+source: tests/snapshots.rs
+expression: output
+---
+| | | |
+| --- | --- | --- |
+| Merged across | | padded |
+| tall | b2 | 3.5 |
+| | b3 | |
diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs
index 6898bb6a..c27d8fe2 100644
--- a/wasm/src/lib.rs
+++ b/wasm/src/lib.rs
@@ -28,6 +28,7 @@ pub enum Format {
Pptx = "pptx",
Rtf = "rtf",
Epub = "epub",
+ Html = "html",
Xlsx = "xlsx",
Ods = "ods",
Odp = "odp",
@@ -45,6 +46,7 @@ impl From