From 398cb28925c6f217ca07e3934b82f9ab17767151 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 20:52:26 +0000 Subject: [PATCH] Make metrics blind to how text was typed (onboarding plan WP-1b) Remove curly_apostrophe_pct, fold quote, dash and ellipsis glyphs in character trigrams, count single quotes alike, and add an invariance suite of eleven transforms. Report version 9.0. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01McJCSUfNJo6GvcAup4iJmV --- CHANGELOG.md | 18 + bench/invariance.py | 317 ++++++++++++++++++ docs/method.md | 42 +++ src/styleprofile/metrics.py | 11 +- src/styleprofile/reports.py | 6 +- src/styleprofile/surface.py | 20 +- tests/snapshots/surface/build-default.txt | 2 +- tests/snapshots/surface/build-demo.txt | 2 +- tests/snapshots/surface/build.txt | 2 +- tests/snapshots/surface/demo.txt | 8 +- tests/snapshots/surface/metrics.txt | 3 +- tests/snapshots/surface/score-against.txt | 6 +- tests/snapshots/surface/score-all.txt | 9 +- .../snapshots/surface/score-by-paragraph.txt | 6 +- .../surface/score-demo-by-paragraph-quiet.txt | 2 +- .../surface/score-demo-by-paragraph.txt | 8 +- tests/snapshots/surface/score-demo-quiet.txt | 2 +- ...core-demo-two-files-by-paragraph-quiet.txt | 2 +- .../score-demo-two-files-by-paragraph.txt | 10 +- tests/snapshots/surface/score-demo.txt | 6 +- tests/snapshots/surface/score-quiet.txt | 2 +- tests/snapshots/surface/score-two-files.txt | 10 +- .../surface/score-writer-and-draft-quiet.txt | 4 +- .../surface/score-writer-and-draft.txt | 12 +- tests/snapshots/surface/score.txt | 6 +- .../surface/show-demo-by-paragraph.txt | 8 +- tests/snapshots/surface/show-newer-minor.txt | 2 +- .../snapshots/surface/show-reference-all.txt | 3 +- tests/snapshots/surface/show-reference.txt | 2 +- .../surface/show-score-writer-and-draft.txt | 12 +- tests/snapshots/surface/show-score.txt | 6 +- tests/snapshots/syntax/build-default.txt | 2 +- tests/snapshots/syntax/build-demo.txt | 2 +- tests/snapshots/syntax/build.txt | 2 +- tests/snapshots/syntax/demo.txt | 8 +- tests/snapshots/syntax/score-against.txt | 6 +- tests/snapshots/syntax/score-all.txt | 9 +- tests/snapshots/syntax/score-by-paragraph.txt | 6 +- .../syntax/score-demo-by-paragraph-quiet.txt | 2 +- .../syntax/score-demo-by-paragraph.txt | 6 +- tests/snapshots/syntax/score-demo-quiet.txt | 2 +- ...core-demo-two-files-by-paragraph-quiet.txt | 4 +- .../score-demo-two-files-by-paragraph.txt | 10 +- tests/snapshots/syntax/score-demo.txt | 4 +- tests/snapshots/syntax/score-quiet.txt | 2 +- tests/snapshots/syntax/score-two-files.txt | 6 +- .../syntax/score-writer-and-draft-quiet.txt | 4 +- .../syntax/score-writer-and-draft.txt | 10 +- tests/snapshots/syntax/score.txt | 6 +- .../syntax/show-demo-by-paragraph.txt | 6 +- tests/snapshots/syntax/show-newer-minor.txt | 2 +- tests/snapshots/syntax/show-reference-all.txt | 3 +- tests/snapshots/syntax/show-reference.txt | 2 +- .../syntax/show-score-writer-and-draft.txt | 10 +- tests/snapshots/syntax/show-score.txt | 6 +- tests/test_build_speed.py | 2 +- tests/test_cli_onboarding.py | 2 +- tests/test_invariance.py | 68 ++++ tests/test_styleprofile.py | 2 - 59 files changed, 594 insertions(+), 149 deletions(-) create mode 100644 bench/invariance.py create mode 100644 tests/test_invariance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b11e7a..6790280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ All notable changes to styleprofile. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the version numbers follow [Semantic Versioning](https://semver.org/); before 1.0, a minor version may break things. +## [Unreleased] + +> [!IMPORTANT] +> **Rebuild your profiles.** References and scores now carry report version 9.0. Different +> majors require rebuilding: run `styleprofile build` again on the writer's texts, and +> `styleprofile score` again for saved score reports. + +### Fixed + +- **Typography no longer changes a verdict.** Converting a writer's straight quotes and + apostrophes to curly ones, as Word, Google Docs and most blog editors do, moved all seven + sample essays from "close" to "somewhat different" or "clearly different". The metric + `curly_apostrophe_pct`, which only recorded how the text was typed, is removed, and + quotation marks, dashes and ellipses are counted alike in either form. A new test suite + applies eleven transforms (quotes, line endings, wrapping, dashes, ellipses, spacing, + paragraph separators, an HTML round trip) and checks the verdicts. See "What styleprofile + ignores" in `docs/method.md`. + ## [0.2.0] - 2026-10-08 The first public release is [available on PyPI](https://pypi.org/project/styleprofile/0.2.0/). diff --git a/bench/invariance.py b/bench/invariance.py new file mode 100644 index 0000000..d0ebd4e --- /dev/null +++ b/bench/invariance.py @@ -0,0 +1,317 @@ +"""Typography invariance: no way of typing or converting a text may change its verdict. + + python bench/invariance.py [--syntax auto|off|on] [--no-medium] + +Each transform in ``TRANSFORMS`` changes how a text is typed, not what it says: curly or +straight quotes, line endings, hard wraps, dash and ellipsis glyphs, spacing, paragraph +separators, an HTML round trip. The same checks run as ``tests/test_invariance.py``; this +script prints the table for a pull request. + +- Writer side: each essay in ``examples/writer`` against a profile built without it, and the + held-out documents of the synthetic ``medium`` corpus (every fifth, sorted) against a + profile built without them. Both verdicts must not change, and Delta may move by at most + ``MAX_DELTA_SHIFT``. +- LLM side: each draft in ``examples/llm-drafts`` against the ``examples/writer`` profile. + Its verdicts must not drop a band, so that normalizing typography can't erase real signal. +""" + +from __future__ import annotations + +import argparse +import html +import re +import sys +import tempfile +import textwrap +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from styleprofile import Profile, Text, build # noqa: E402 +from styleprofile.core import DISTANCES, LikenessVerdict # noqa: E402 + +MAX_DELTA_SHIFT = 0.25 +SYNTAX: dict[str, bool | Literal["auto"]] = {"auto": "auto", "off": False, "on": True} +LIKENESS_LEVELS = ( + LikenessVerdict.LIKE_REFERENCE, + LikenessVerdict.FEW_TRAITS, + LikenessVerdict.LEANS, + LikenessVerdict.LIKE_DRAFTS, +) + + +def _curly(text: str) -> str: + text = re.sub(r"(\w)'(\w)", "\\1\u2019\\2", text) + text = re.sub(r"(^|\s)'", "\\1\u2018", text) + text = text.replace("'", "\u2019") + text = re.sub(r'(^|\s)"', "\\1\u201c", text) + return text.replace('"', "\u201d") + + +def _straight(text: str) -> str: + for curly, straight in ( + ("\u2018", "'"), + ("\u2019", "'"), + ("\u201c", '"'), + ("\u201d", '"'), + ): + text = text.replace(curly, straight) + return text + + +def _paragraphs(text: str, change: Callable[[str], str]) -> str: + """Apply ``change`` to each plain paragraph, leaving headings, lists and code alone.""" + parts = re.split(r"(\n\s*\n)", text) + return "".join( + part if not part.strip() or re.match(r"\s*([#>*+-]|\d+\.|```|\|)", part) else change(part) + for part in parts + ) + + +def _wrap(text: str) -> str: + return _paragraphs(text, lambda p: textwrap.fill(p, 72, break_long_words=False)) + + +def _inline(text: str) -> str: + text = html.escape(text, quote=False) + text = re.sub(r"(\*\*|__)(.+?)\1", r"\2", text) + return re.sub(r"(?\2", text) + + +def _html_block(block: str) -> list[str]: + out: list[str] = [] + paragraph: list[str] = [] + items: list[str] = [] + ordered = False + + def flush() -> None: + if paragraph: + out.append("

" + _inline("\n".join(paragraph)) + "

") + paragraph.clear() + if items: + tag = "ol" if ordered else "ul" + out.append(f"<{tag}>" + "".join(f"
  • {_inline(i)}
    • " for i in items) + f"") + items.clear() + + for line in block.split("\n"): + heading = re.match(r"(#{1,6})\s+(.*)", line) + item = re.match(r"\s*(?:([-*+])|(\d+\.))\s+(.*)", line) + if heading: + flush() + level = len(heading.group(1)) + out.append(f"{_inline(heading.group(2))}") + elif item: + if paragraph: + flush() + ordered = item.group(2) is not None + items.append(item.group(3)) + else: + if items: + flush() + paragraph.append(line) + flush() + return out + + +def _html(text: str) -> str: + """Markdown rendered to simple HTML: paragraphs, headings, lists and emphasis.""" + blocks = [part for block in re.split(r"\n\s*\n", text.strip()) for part in _html_block(block)] + return "\n" + "\n".join(blocks) + "\n\n" + + +def _sentence_end(spacing: str) -> Callable[[str], str]: + return lambda text: re.sub( + r"([.!?][\"'\u201d\u2019)]?) (?=[A-Z\u201c\"'])", r"\1" + spacing, text + ) + + +@dataclass(frozen=True) +class Transform: + name: str + change: Callable[[str], str] + input_format: str = "auto" + # Removing paragraph breaks leaves the paragraph metrics out (they would be one huge + # paragraph), which can only make a text look closer; its Delta may fall, but not rise. + drops_metrics: bool = False + + +TRANSFORMS: tuple[Transform, ...] = ( + Transform("straight to typographic quotes", _curly), + Transform("typographic to straight quotes", _straight), + Transform("CRLF line endings", lambda t: t.replace("\n", "\r\n")), + Transform("hard wrap at 72 columns", _wrap), + Transform("em dash to --", lambda t: t.replace("\u2014", "--")), + Transform("-- to em dash", lambda t: re.sub(r"(?<=\w)--(?=\w)", "\u2014", t)), + Transform("... to ellipsis glyph", lambda t: t.replace("...", "\u2026")), + Transform("ellipsis glyph to ...", lambda t: t.replace("\u2026", "...")), + Transform("two spaces after sentences", _sentence_end(" ")), + Transform("non-breaking space after sentences", _sentence_end("\u00a0")), + Transform( + "single newline between paragraphs", + lambda t: re.sub(r"\n\s*\n", "\n", t), + drops_metrics=True, + ), + Transform( + "blank lines removed", + lambda t: "\n".join(x for x in t.split("\n") if x.strip()), + drops_metrics=True, + ), + Transform("Markdown to HTML and back", _html, input_format="html"), + Transform("trailing spaces and tabs", lambda t: re.sub(r"(?m)$", " \t", t)), +) + + +@dataclass(frozen=True) +class Case: + side: str # "writer" or "llm" + name: str + text: str + profile: Profile + + +@dataclass(frozen=True) +class Outcome: + transform: str + side: str + cases: int + failures: list[str] + worst_shift: float + + +def _read(folder: Path) -> list[Text]: + return [ + Text(path.read_text(encoding="utf-8"), path.name) for path in sorted(folder.glob("*.md")) + ] + + +def _score(profile: Profile, text: str, syntax: str, input_format: str = "auto"): + """Score ``text`` as the command line would: from a file, so line endings and HTML are + read the way they are for a user's own files.""" + flag = SYNTAX[syntax] + suffix = ".html" if input_format == "html" else ".md" + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / f"doc{suffix}" + with open(path, "w", encoding="utf-8", newline="") as handle: + handle.write(text) + return profile.score(path, syntax=flag, input_format=input_format) + + +def _levels(result) -> tuple[int, int, float | None]: + verdict = result.verdict + likeness = result.likeness_verdict + distance = DISTANCES.index(verdict) if verdict in DISTANCES else -1 + like = LIKENESS_LEVELS.index(likeness) if likeness in LIKENESS_LEVELS else -1 + return distance, like, result.delta + + +def cases(syntax: str, medium: bool = True) -> list[Case]: + flag = SYNTAX[syntax] + writer = _read(ROOT / "examples" / "writer") + contrast = _read(ROOT / "examples" / "llm-drafts") + out: list[Case] = [] + for held in writer: + rest = [text for text in writer if text is not held] + profile = build(rest, contrast=contrast, syntax=flag) + out.append(Case("writer", held.name or "", held.text, profile)) + whole = build(writer, contrast=contrast, syntax=flag) + out += [Case("llm", text.name or "", text.text, whole) for text in contrast] + if medium: + out += _medium(syntax) + return out + + +def _medium(syntax: str) -> list[Case]: + import gen + + flag = SYNTAX[syntax] + folder = gen.generate("medium") + documents = sorted((folder / "writer").glob("*.md")) + held = documents[::5] + kept = [path for path in documents if path not in held] + contrast = sorted((folder / "contrast").glob("*.md")) + profile = build( + [Text(path.read_text(encoding="utf-8"), path.name) for path in kept], + contrast=[Text(path.read_text(encoding="utf-8"), path.name) for path in contrast], + syntax=flag, + ) + return [Case("writer", path.name, path.read_text(encoding="utf-8"), profile) for path in held] + + +def run(syntax: str = "auto", medium: bool = True) -> list[Outcome]: + all_cases = cases(syntax, medium) + base = { + (case.side, case.name): _levels(_score(case.profile, case.text, syntax)) + for case in all_cases + } + outcomes = [] + for transform in TRANSFORMS: + for side in ("writer", "llm"): + failures: list[str] = [] + worst = 0.0 + chosen = [case for case in all_cases if case.side == side] + for case in chosen: + changed = transform.change(case.text) + after = _levels(_score(case.profile, changed, syntax, transform.input_format)) + distance, like, delta = base[(case.side, case.name)] + new_distance, new_like, new_delta = after + moved = (new_delta or 0.0) - (delta or 0.0) + shift = abs(moved) + counted = max(moved, 0.0) if transform.drops_metrics else shift + if side == "writer": + worst = max(worst, shift) + if transform.drops_metrics: + bad = ( + new_distance > distance or new_like > like or counted > MAX_DELTA_SHIFT + ) + else: + bad = (new_distance, new_like) != ( + distance, + like, + ) or counted > MAX_DELTA_SHIFT + else: + bad = new_distance < distance or new_like < like + if bad: + failures.append( + f"{case.name}: {DISTANCES[distance] if distance >= 0 else '-'}" + f"/{like} to {new_distance}/{new_like}, Delta " + f"{delta if delta is None else round(delta, 2)} to " + f"{new_delta if new_delta is None else round(new_delta, 2)}" + ) + outcomes.append(Outcome(transform.name, side, len(chosen), failures, worst)) + return outcomes + + +def table(outcomes: list[Outcome]) -> str: + lines = [ + "| Transform | Side | Cases | Failing | Largest Delta shift |", + "|---|---|---|---|---|", + ] + for item in outcomes: + shift = f"{item.worst_shift:.2f}" if item.side == "writer" else "-" + lines.append( + f"| {item.transform} | {item.side} | {item.cases} | {len(item.failures)} | {shift} |" + ) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=(__doc__ or "").split("\n\n")[0]) + parser.add_argument("--syntax", choices=("auto", "off", "on"), default="auto") + parser.add_argument("--no-medium", action="store_true", help="skip the synthetic corpus") + parser.add_argument("--details", action="store_true", help="list each failing case") + args = parser.parse_args() + outcomes = run(args.syntax, not args.no_medium) + print(table(outcomes)) + if args.details: + for item in outcomes: + for failure in item.failures: + print(f"{item.transform} [{item.side}] {failure}") + sys.exit(1 if any(item.failures for item in outcomes) else 0) + + +if __name__ == "__main__": + main() diff --git a/docs/method.md b/docs/method.md index bc07247..53d6151 100644 --- a/docs/method.md +++ b/docs/method.md @@ -738,3 +738,45 @@ contain fewer than 5% of fifty common English function words. It is deliberately conservative: it does not identify a language or refuse text. Names, specialist vocabulary and mixed languages can confuse it. English metrics and the optional English parser can be unreliable on other languages. The check runs on original inputs for build and score. + +## What styleprofile ignores + +How a text was typed or converted is not style. These changes to a text must not change its +verdicts, so a writer's essays read the same pasted from Word, saved from a blog editor, or +read from a rendered page: + +- straight quotes and apostrophes turned typographic, and the reverse; +- CRLF line endings, and hard wrapping within paragraphs; +- `—` written `--`, and the reverse; `...` written `…`, and the reverse; +- two spaces, or a non-breaking space, after a sentence; +- trailing spaces and tabs; +- Markdown rendered to simple HTML (paragraphs, headings, lists, emphasis) and read back + with `--input-format html`. + +Two more changes remove information rather than re-type it, and may only make a text look +closer, never further: paragraph breaks written as one newline, and blank lines removed. A +text with no paragraph breaks has no paragraph evidence, so those metrics are left out (see +"Input structure and language"), and the rest of the comparison stands. + +Three things make this hold: + +- Counts treat the glyphs alike. Quotations count opening curly double quotes and straight + pairs, apostrophes count `'`, `‘` and `’`, em dashes and ellipses count either form, and + every word list reads a curly apostrophe as a straight one. Character trigrams fold quotes, + dashes and ellipses to one form. +- Nothing is scored that only records encoding. `curly_apostrophe_pct`, the share of + apostrophes that were curly, is removed rather than listed as "not scored": it records the + editor a text passed through, not the writing, so it only ever added a false alarm to a + writer's converted text, and a metric nobody scores would still invite a reader to ask what it means. Removing + it leaves the contrast AUC where it was (1.0 on `examples/` and on the `medium` corpus, + with and without spaCy), and the sample LLM drafts score the same with their quotes + straightened. +- No score is capped to make up for a metric that misbehaves. A cap on z cost detection of + short LLM text and widened the writer's own range when it was tried, so every transform has + to be fixed where the metric reads the text. + +`tests/test_invariance.py` applies each transform to every essay in `examples/writer` (against +a profile built without it), to the 40 held-out documents of the synthetic `medium` corpus +(every fifth, sorted), and to the sample LLM drafts, with and without spaCy. A writer's +verdicts must not change and its Delta may move by at most 0.25; an LLM draft's verdicts must +not move toward the writer. `python bench/invariance.py` prints the table. diff --git a/src/styleprofile/metrics.py b/src/styleprofile/metrics.py index 53810a0..9c3cbef 100644 --- a/src/styleprofile/metrics.py +++ b/src/styleprofile/metrics.py @@ -96,7 +96,6 @@ class Count(StrEnum): WORDS = "words" SENTENCES = "sentences" PARAGRAPHS = "paragraphs" - APOSTROPHES = "apostrophes" @dataclass(frozen=True) @@ -120,7 +119,7 @@ def resolution(self, counts: Mapping[str, float]) -> float: """Half of one occurrence per chunk, in this metric's units: the smallest meaningful spread. - ``counts`` are the reference's typical words, sentences, paragraphs and apostrophes + ``counts`` are the reference's typical words, sentences and paragraphs per chunk. A rate per 1,000 words moves by 1000 / words per occurrence; a percentage by 100 / (whatever it is a share of). Other metrics have no count resolution. """ @@ -316,14 +315,6 @@ def _opener(name: str, label: str, about: str) -> Metric: "Quotations", "Quotations: opening curly double quotes, or pairs of straight ones.", ), - Metric( - "punctuation", - "curly_apostrophe_pct", - "Curly apostrophes (vs straight)", - PCT, - Count.APOSTROPHES, - about="Share of apostrophes that are curly.", - ), _rate( "voice", "contractions_per_1k", diff --git a/src/styleprofile/reports.py b/src/styleprofile/reports.py index 9aa1e69..7831aa2 100644 --- a/src/styleprofile/reports.py +++ b/src/styleprofile/reports.py @@ -41,9 +41,11 @@ # 7 (0.2.0): nominalizations_per_1k no longer counts words such as fence, city or sentence # (see ``syntax.is_nominalization``), so a version 6 profile's values would not match. # 8 (0.2.0): long plain blocks split at sentences; long single paragraphs omit structure. -VERSION = 8 +# 9: typography never changes a verdict: curly_apostrophe_pct is gone (it recorded how the +# text was typed), and trigrams and the apostrophe count fold quote glyphs. +VERSION = 9 # Additive report fields increment this; legacy integer versions have minor zero. -MINOR_VERSION = 1 +MINOR_VERSION = 0 # The version of evaluation reports (``styleprofile evaluate``), counted separately. EVALUATION_VERSION = 2 REFERENCE: Final = "reference" diff --git a/src/styleprofile/surface.py b/src/styleprofile/surface.py index 8e704ef..921ded3 100644 --- a/src/styleprofile/surface.py +++ b/src/styleprofile/surface.py @@ -80,6 +80,7 @@ _ABBREVIATION = re.compile(r"\b(?:e\.g|i\.e|vs|mr|mrs|ms|dr|st|jr|sr|fig|approx)\.$", re.I) # Abbreviations that are also ordinary words, only before a number ("No. 5", not "said no."). _NUMBER_ABBREVIATION = re.compile(r"\b(?:No|Nos|[Vv]ol|pp|[Cc]h)\.$") +_SINGLE_QUOTES = ("'", "\N{LEFT SINGLE QUOTATION MARK}", "\N{RIGHT SINGLE QUOTATION MARK}") _EM_DASH = re.compile(r"\N{EM DASH}|(?<=\w)--(?=\w)|(?<=\w) -{1,2} (?=\w)") _ELLIPSIS = re.compile(r"\.\.\.|\N{HORIZONTAL ELLIPSIS}") _FUNCTION_SET = frozenset(FUNCTION_WORDS) @@ -536,7 +537,7 @@ def surface_metrics( mean_length = _mean(lengths) length_sd = statistics.pstdev(lengths) if lengths else None - apostrophes = text.count("'") + text.count("\N{RIGHT SINGLE QUOTATION MARK}") + apostrophes = sum(text.count(mark) for mark in _SINGLE_QUOTES) straight_double = text.count('"') function_counts = Counter(token for token in tokens if token in _FUNCTION_SET) @@ -581,7 +582,6 @@ def rate(count: int) -> float | None: "quotations_per_1k": rate( text.count("\N{LEFT DOUBLE QUOTATION MARK}") + straight_double // 2 ), - "curly_apostrophe_pct": _pct(text.count("\N{RIGHT SINGLE QUOTATION MARK}"), apostrophes), "contractions_per_1k": rate( sum( token in S_CONTRACTIONS or bool(CONTRACTION_SUFFIX.search(token)) @@ -632,9 +632,23 @@ def masked_bigrams(text: str, tokens: list[str] | None = None) -> Counter[str]: ) +# Glyphs that only record how a text was typed, folded to one form so trigrams don't count them. +_TRIGRAM_GLYPHS = str.maketrans( + { + "\N{LEFT SINGLE QUOTATION MARK}": "'", + "\N{RIGHT SINGLE QUOTATION MARK}": "'", + "\N{LEFT DOUBLE QUOTATION MARK}": '"', + "\N{RIGHT DOUBLE QUOTATION MARK}": '"', + "\N{EM DASH}": "-", + "\N{EN DASH}": "-", + "\N{HORIZONTAL ELLIPSIS}": "...", + } +) + + def char_trigrams(text: str) -> Counter[str]: """Character trigrams of prose text (``prose(markdown).text``), case-folded.""" - text = re.sub(r"\s+", " ", text.casefold()) + text = re.sub(r"\s+", " ", text.casefold().translate(_TRIGRAM_GLYPHS)) return Counter(map("".join, zip(text, text[1:], text[2:], strict=False))) diff --git a/tests/snapshots/surface/build-default.txt b/tests/snapshots/surface/build-default.txt index 4e72fd3..82aef39 100644 --- a/tests/snapshots/surface/build-default.txt +++ b/tests/snapshots/surface/build-default.txt @@ -3,7 +3,7 @@ $ styleprofile build '/writer' --no-syntax --- stdout STYLE PROFILE 7 chunks, 4,496 words -As a reference held-out Delta 0.68 typically, 95% under 0.89 (7 documents) +As a reference held-out Delta 0.70 typically, 95% under 0.90 (7 documents) Shorter texts: calibrated at 75 words (124 pieces), 150 words (56 pieces); not at 300 words (28 pieces from 7 documents, worth 18 independent ones), as a length needs 20 independent pieces from 3 or more documents. Under 75 words, no verdict. diff --git a/tests/snapshots/surface/build-demo.txt b/tests/snapshots/surface/build-demo.txt index ef96580..ae650b2 100644 --- a/tests/snapshots/surface/build-demo.txt +++ b/tests/snapshots/surface/build-demo.txt @@ -3,7 +3,7 @@ $ styleprofile build --by-paragraph '/demo/writer' --contrast '/demo/c --- stdout STYLE PROFILE 55 chunks, 29,312 words -As a reference held-out Delta 0.53 typically, 95% under 0.84 (40 documents) +As a reference held-out Delta 0.54 typically, 95% under 0.85 (40 documents) Shorter texts: calibrated at 75 words (393 pieces), 150 words (197 pieces), 300 words (85 pieces). Under 75 words, no verdict. diff --git a/tests/snapshots/surface/build.txt b/tests/snapshots/surface/build.txt index 5ebb741..15e5035 100644 --- a/tests/snapshots/surface/build.txt +++ b/tests/snapshots/surface/build.txt @@ -3,7 +3,7 @@ $ styleprofile build examples/writer --contrast examples/llm-drafts -o '/wr --- stdout STYLE PROFILE 7 chunks, 4,496 words -As a reference held-out Delta 0.68 typically, 95% under 0.89 (7 documents) +As a reference held-out Delta 0.70 typically, 95% under 0.90 (7 documents) Shorter texts: calibrated at 75 words (124 pieces), 150 words (56 pieces); not at 300 words (28 pieces from 7 documents, worth 18 independent ones), as a length needs 20 independent pieces from 3 or more documents. Under 75 words, no verdict. diff --git a/tests/snapshots/surface/demo.txt b/tests/snapshots/surface/demo.txt index e20aa60..430b2c5 100644 --- a/tests/snapshots/surface/demo.txt +++ b/tests/snapshots/surface/demo.txt @@ -3,7 +3,7 @@ $ styleprofile demo --dir '/bundled-demo' --no-syntax --- stdout STYLE PROFILE 7 chunks, 4,496 words -As a reference held-out Delta 0.68 typically, 95% under 0.89 (7 documents) +As a reference held-out Delta 0.70 typically, 95% under 0.90 (7 documents) Shorter texts: calibrated at 75 words (124 pieces), 150 words (56 pieces); not at 300 words (28 pieces from 7 documents, worth 18 independent ones), as a length needs 20 independent pieces from 3 or more documents. Under 75 words, no verdict. @@ -21,9 +21,9 @@ Next, score a draft against it: styleprofile score /bundled-demo/writer.profile.json STYLE COMPARISON 1 chunk, 542 words vs writer.profile.json (7 chunks) -Overall: close Delta 0.85 - Lower is closer. The reference's own held-out writing at this length (542 words) scores 0.71 typically (95% under - 0.99). +Overall: close Delta 0.87 + Lower is closer. The reference's own held-out writing at this length (542 words) scores 0.73 typically (95% under + 1.00). LLM-likeness: like the reference 0.71 The reference's own writing at this length (542 words) scores 0.20 typically (95% under 0.86); the LLM drafts 14.20. diff --git a/tests/snapshots/surface/metrics.txt b/tests/snapshots/surface/metrics.txt index 4dd5840..5b64792 100644 --- a/tests/snapshots/surface/metrics.txt +++ b/tests/snapshots/surface/metrics.txt @@ -1,7 +1,7 @@ $ styleprofile metrics [exit 0] --- stdout -153 metrics. Syntax and sentence-opening metrics need spaCy. +152 metrics. Syntax and sentence-opening metrics need spaCy. Size Words Prose words, after code, tables and markup are removed. @@ -41,7 +41,6 @@ Punctuation Ellipses /1k Ellipses (... or the single glyph). Quotations /1k Quotations: opening curly double quotes, or pairs of straight ones. - Curly apostrophes (vs straight) % Share of apostrophes that are curly. Voice and stance Contractions /1k Contractions such as don't, we're and it's (not possessives). diff --git a/tests/snapshots/surface/score-against.txt b/tests/snapshots/surface/score-against.txt index b46e5f6..1c22c8e 100644 --- a/tests/snapshots/surface/score-against.txt +++ b/tests/snapshots/surface/score-against.txt @@ -5,9 +5,9 @@ Reference: built from 7 documents (4,496 words) in