diff --git a/src/ebookmaker/parsers/GutenbergTextParser.py b/src/ebookmaker/parsers/GutenbergTextParser.py index af1595a..b7e6cf3 100644 --- a/src/ebookmaker/parsers/GutenbergTextParser.py +++ b/src/ebookmaker/parsers/GutenbergTextParser.py @@ -14,6 +14,7 @@ from __future__ import unicode_literals import importlib +import os import re import six @@ -35,6 +36,43 @@ MAX_BEFORE = 5 # no. of empty lines that mark a

+# A paragraph in which *every* line starts with at least this many spaces was +# hand-indented and hand-wrapped by the transcriber, so its line breaks are +# intentional: keep them instead of reflowing the paragraph. +# +# Without this, verse is recognized mainly by every line starting with a +# capital letter. Languages that do not capitalize every verse line +# (Finnish, Swedish, German, ...) lose: the penalty grows with the number of +# lines, so long stanzas were always reflowed into a prose block quote. +# +# Set to 0 to restore the old behaviour. Can be overridden per run with the +# environment variable EBOOKMAKER_VERSE_INDENT. +try: + VERSE_INDENT = int(os.environ.get('EBOOKMAKER_VERSE_INDENT', 4)) +except ValueError: + VERSE_INDENT = 4 + +# Strip the common leading indentation from a preformatted (verse) block and +# express it as a CSS margin instead. The transcriber's indentation is only a +# marker; rendering it as hard   characters makes the block start with +# stray spaces and prevents it from adapting to the reader's screen. Relative +# indentation *inside* the block (a line indented deeper than its siblings) is +# always preserved. +VERSE_STRIP_INDENT = True + +# The margin that replaces the stripped indentation. It is deliberately the +# same for every block: a poem whose stanzas were indented by slightly +# different amounts should still line up, and a title page centered with 30 +# spaces should not end up pushed a third of the way across the screen. +VERSE_MARGIN = 5 # per cent + +# A block whose longest line falls short of this fraction of the width the +# transcription is filled to was broken by hand, not by a word wrapper. +# Reflowed prose always fills its lines nearly to the margin, so a block that +# never comes close was typed one line at a time: a publisher's imprint, a +# cast of characters, a table of contents, an address, a signature. +SHORT_BLOCK_RATIO = 0.66 + RE_ITALICS = re.compile(r"\b_([^_]+?)_\b") RE_INDENT = re.compile(r"^\s+") @@ -175,7 +213,19 @@ def __init__(self, par): self.indents = list(map(self._indent, lines)) self.titles = list(map(self._istitle, lines)) + self.numbers = list(map(self._isnumbered, lines)) + if most(self.numbers): + # A numbered list (table of contents, numbered verse, dated + # entries) is hand-broken just like a capitalized one. Roman + # numerals always counted as titles because they are letters; + # arabic numerals did not, so "1. 2. 3." was reflowed while + # "I. II. III." was not. Only applied when most lines are + # numbered, so that a prose line happening to start with a + # number ("...Apartment / 60.") is not mistaken for a list. + self.titles = or_(self.titles, self.numbers) self.uppers = list(map(six.text_type.isupper, lines)) + # lines starting with VERSE_INDENT or more spaces + self.indenteds = [i >= VERSE_INDENT for i in self.indents] # skip last line, which is almost always shorter self.length = MinMaxAvg(self.lengths[:-1]) @@ -210,6 +260,12 @@ def _istitle(line): m = re.search(r'\w', line) return m and m.group(0).isupper() + @staticmethod + def _isnumbered(line): + """ Return True if the line starts with a number. """ + m = re.search(r'\w', line) + return bool(m) and m.group(0).isdigit() + def _rhyme_stemmer(self, line): """ Return the stem of the rhyme. @@ -261,7 +317,12 @@ def __init__(self): self.after = 0 self.id = None self.prev = None + self.next = None self.debug_message = '' + self.force_verse = False + self.short_block = False + self.strip_indent = 0 + self.fill_width = 0 self.scores = Struct() for subject in SUBJECTS: @@ -318,6 +379,30 @@ def long_lines(self): # same indentation pattern as pars before and after + def deeply_indented(self): + """ Return True if *every* line starts with VERSE_INDENT+ spaces. """ + + if not VERSE_INDENT: + return False + return bool(self.metrics.indenteds) and all(self.metrics.indenteds) + + def is_block(self): + """ Return True if this par is shipped as a hand-broken block. """ + + return self.force_verse or self.scores.quote > THRESHOLD + + def is_flowed_prose(self): + """ Return True if this par is an ordinary, re-wrapped prose par. + + Only such a par wants the first-line indent that the stylesheet gives + to every

. Hand-broken blocks (verse, title pages, tables of + contents) do not: their first line would start with a stray indent + that the transcription does not have. + """ + + return not (self.is_block() or self.short_block or + self.scores.header > THRESHOLD) + def header_smells(self): """ Test some words we know hint at headers """ return RE_HEADER_SMELLS.findall(" ".join(self.lines)) @@ -370,6 +455,21 @@ def analyze(self): self.msg("any header smells") self.scores.header *= 2.0 + # hand-indented block: never reflow + + if not any(self.p_smells()): + if self.deeply_indented(): + self.msg("every line indented >= %d" % VERSE_INDENT) + self.force_verse = True + + elif (self.fill_width and self.metrics.lengths and + max(self.metrics.lengths) < self.fill_width * SHORT_BLOCK_RATIO): + self.msg("no line reaches %d of %d columns" % ( + max(self.metrics.lengths), self.fill_width)) + self.short_block = True + if self.metrics.cnt_lines > 1: + self.force_verse = True + # analyze indentation if half(self.metrics.indents): @@ -469,14 +569,6 @@ def __init__(self, attribs=None): self.body = 0 self.max_blanks = 0 self.pars = [] - self.text = "" - self.pg_header = "" - self.pg_footer = "" - - - def unicode_content(self): - return self.pg_header + self.text + self.pg_footer - def get_charset_from_meta(self): """ Parse text for hints about charset. """ @@ -507,6 +599,14 @@ def analyze(self): last_par.next = par last_par = par + # The width the transcription is filled to. Taken as a high + # percentile rather than the maximum, so that one stray long line + # does not set the standard for the whole book. + lengths = sorted(len(line) for par in self.pars for line in par.lines) + fill_width = lengths[int(len(lengths) * 0.9)] if lengths else 0 + for par in self.pars: + par.fill_width = fill_width + for par in self.pars: par.analyze() @@ -536,9 +636,30 @@ def analyze(self): level = max(MAX_BEFORE - par.before, 0) par.tag = "h%d" % (level + 1) else: - if par.scores.quote > THRESHOLD: - if par.scores.verse > 1.0: + # Kill the stylesheet's first-line indent on blocks that were + # broken by hand. The stylesheet only suppresses it after a + # heading, so the *first* stanza of a poem looked right while + # every following stanza -- and every line of a title page or + # a table of contents -- started with a stray indent. + if par.is_block(): + par.styles['text-indent'] = '0' + elif par.short_block or par.metrics.cnt_lines == 1: + # A lone line is ambiguous: in a novel it is a one line + # prose paragraph and keeps its indent; between headings + # and hand-broken blocks it is front matter -- a byline, + # an imprint, a section title, a numbered subtitle. + neighbours = [p for p in (par.prev, par.next) if p] + if not any(p.is_flowed_prose() for p in neighbours): + par.styles['text-indent'] = '0' + + if par.is_block(): + # force_verse: hand-indented, keep the line breaks + if par.force_verse or par.scores.verse > 1.0: par.styles['white-space'] = 'pre' + if VERSE_STRIP_INDENT and par.metrics.indents: + par.strip_indent = min(par.metrics.indents) + if par.strip_indent: + par.styles['margin-left'] = '%d%%' % VERSE_MARGIN else: par.styles['margin-left'] = '%d%%' % ( par.metrics.indent.first * 100 / 72) @@ -572,6 +693,15 @@ def it_repl(matchobj): return RE_ITALICS.sub(it_repl, s) if par.styles.get('white-space', '') == 'pre': + if par.strip_indent: + n = par.strip_indent + par.lines = [line[n:] for line in par.lines] + if par.lines: + # A block never starts indented relative to its own body: + # the blank line above it is what separates it from the + # previous paragraph. Indentation on *later* lines is + # meaningful and is kept. + par.lines[0] = par.lines[0].lstrip(' ') par.lines = map(self.preformat, par.lines) del par.styles['white-space'] @@ -633,13 +763,13 @@ def parse(self): if self.xhtml is not None: return - text = HTMLParserBase.unicode_content(self) - self.text, self.pg_header, self.pg_footer = strip_headers_from_txt(text) - if 'x-header' in self.pg_header and options.production: + text = self.unicode_content() + text, pg_header, pg_footer = strip_headers_from_txt(text) + if 'x-header' in pg_header and options.production: error('header marker is missing in %s', self.attribs.url) - if 'x-header' in self.pg_footer and options.production: + if 'x-header' in pg_footer and options.production: error('footer marker is missing in %s', self.attribs.url) - text = self.text + text = parsers.RE_RESTRICTED.sub('', text) text = gg.xmlspecialchars(text) @@ -692,18 +822,12 @@ def parse(self): for body in xpath(self.xhtml, '//xhtml:body'): xhtmlparser = lxml.html.XHTMLParser(huge_tree=True) - pg_header_pre = etree.Element(NS.xhtml.pre) - pg_header_pre.attrib['id'] = 'pg-header' - pg_header_pre.text = self.pg_header - body.append(pg_header_pre) + body.append(etree.fromstring(pg_header, xhtmlparser)) for par in self.pars: p = etree.fromstring(self.ship_out(par), xhtmlparser) p.tail = '\n\n' body.append(p) - pg_footer_pre = etree.Element(NS.xhtml.pre) - pg_footer_pre.text = self.pg_footer - pg_footer_pre.attrib['id'] = 'pg-footer' - body.append(pg_footer_pre) + body.append(etree.fromstring(pg_footer, xhtmlparser)) self.pars = []