diff --git a/src/ebookmaker/parsers/GutenbergTextParser.py b/src/ebookmaker/parsers/GutenbergTextParser.py index af1595a..b7e6cf3 100644 --- a/src/ebookmaker/parsers/GutenbergTextParser.py +++ b/src/ebookmaker/parsers/GutenbergTextParser.py @@ -14,6 +14,7 @@ from __future__ import unicode_literals import importlib +import os import re import six @@ -35,6 +36,43 @@ MAX_BEFORE = 5 # no. of empty lines that mark a
. Hand-broken blocks (verse, title pages, tables of + contents) do not: their first line would start with a stray indent + that the transcription does not have. + """ + + return not (self.is_block() or self.short_block or + self.scores.header > THRESHOLD) + def header_smells(self): """ Test some words we know hint at headers """ return RE_HEADER_SMELLS.findall(" ".join(self.lines)) @@ -370,6 +455,21 @@ def analyze(self): self.msg("any header smells") self.scores.header *= 2.0 + # hand-indented block: never reflow + + if not any(self.p_smells()): + if self.deeply_indented(): + self.msg("every line indented >= %d" % VERSE_INDENT) + self.force_verse = True + + elif (self.fill_width and self.metrics.lengths and + max(self.metrics.lengths) < self.fill_width * SHORT_BLOCK_RATIO): + self.msg("no line reaches %d of %d columns" % ( + max(self.metrics.lengths), self.fill_width)) + self.short_block = True + if self.metrics.cnt_lines > 1: + self.force_verse = True + # analyze indentation if half(self.metrics.indents): @@ -469,14 +569,6 @@ def __init__(self, attribs=None): self.body = 0 self.max_blanks = 0 self.pars = [] - self.text = "" - self.pg_header = "" - self.pg_footer = "" - - - def unicode_content(self): - return self.pg_header + self.text + self.pg_footer - def get_charset_from_meta(self): """ Parse text for hints about charset. """ @@ -507,6 +599,14 @@ def analyze(self): last_par.next = par last_par = par + # The width the transcription is filled to. Taken as a high + # percentile rather than the maximum, so that one stray long line + # does not set the standard for the whole book. + lengths = sorted(len(line) for par in self.pars for line in par.lines) + fill_width = lengths[int(len(lengths) * 0.9)] if lengths else 0 + for par in self.pars: + par.fill_width = fill_width + for par in self.pars: par.analyze() @@ -536,9 +636,30 @@ def analyze(self): level = max(MAX_BEFORE - par.before, 0) par.tag = "h%d" % (level + 1) else: - if par.scores.quote > THRESHOLD: - if par.scores.verse > 1.0: + # Kill the stylesheet's first-line indent on blocks that were + # broken by hand. The stylesheet only suppresses it after a + # heading, so the *first* stanza of a poem looked right while + # every following stanza -- and every line of a title page or + # a table of contents -- started with a stray indent. + if par.is_block(): + par.styles['text-indent'] = '0' + elif par.short_block or par.metrics.cnt_lines == 1: + # A lone line is ambiguous: in a novel it is a one line + # prose paragraph and keeps its indent; between headings + # and hand-broken blocks it is front matter -- a byline, + # an imprint, a section title, a numbered subtitle. + neighbours = [p for p in (par.prev, par.next) if p] + if not any(p.is_flowed_prose() for p in neighbours): + par.styles['text-indent'] = '0' + + if par.is_block(): + # force_verse: hand-indented, keep the line breaks + if par.force_verse or par.scores.verse > 1.0: par.styles['white-space'] = 'pre' + if VERSE_STRIP_INDENT and par.metrics.indents: + par.strip_indent = min(par.metrics.indents) + if par.strip_indent: + par.styles['margin-left'] = '%d%%' % VERSE_MARGIN else: par.styles['margin-left'] = '%d%%' % ( par.metrics.indent.first * 100 / 72) @@ -572,6 +693,15 @@ def it_repl(matchobj): return RE_ITALICS.sub(it_repl, s) if par.styles.get('white-space', '') == 'pre': + if par.strip_indent: + n = par.strip_indent + par.lines = [line[n:] for line in par.lines] + if par.lines: + # A block never starts indented relative to its own body: + # the blank line above it is what separates it from the + # previous paragraph. Indentation on *later* lines is + # meaningful and is kept. + par.lines[0] = par.lines[0].lstrip(' ') par.lines = map(self.preformat, par.lines) del par.styles['white-space'] @@ -633,13 +763,13 @@ def parse(self): if self.xhtml is not None: return - text = HTMLParserBase.unicode_content(self) - self.text, self.pg_header, self.pg_footer = strip_headers_from_txt(text) - if 'x-header' in self.pg_header and options.production: + text = self.unicode_content() + text, pg_header, pg_footer = strip_headers_from_txt(text) + if 'x-header' in pg_header and options.production: error('header marker is missing in %s', self.attribs.url) - if 'x-header' in self.pg_footer and options.production: + if 'x-header' in pg_footer and options.production: error('footer marker is missing in %s', self.attribs.url) - text = self.text + text = parsers.RE_RESTRICTED.sub('', text) text = gg.xmlspecialchars(text) @@ -692,18 +822,12 @@ def parse(self): for body in xpath(self.xhtml, '//xhtml:body'): xhtmlparser = lxml.html.XHTMLParser(huge_tree=True) - pg_header_pre = etree.Element(NS.xhtml.pre) - pg_header_pre.attrib['id'] = 'pg-header' - pg_header_pre.text = self.pg_header - body.append(pg_header_pre) + body.append(etree.fromstring(pg_header, xhtmlparser)) for par in self.pars: p = etree.fromstring(self.ship_out(par), xhtmlparser) p.tail = '\n\n' body.append(p) - pg_footer_pre = etree.Element(NS.xhtml.pre) - pg_footer_pre.text = self.pg_footer - pg_footer_pre.attrib['id'] = 'pg-footer' - body.append(pg_footer_pre) + body.append(etree.fromstring(pg_footer, xhtmlparser)) self.pars = []