diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8e7189..61c2cb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [master] + branches: [master, dev] pull_request: - branches: [master] + branches: [master, dev] env: CARGO_TERM_COLOR: always @@ -64,6 +64,32 @@ jobs: - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends lilypond - run: cargo test --test lilypond_oracle -- --ignored --nocapture + lilypond-corpus: + # The grammar against LilyPond's own regression tests and snippets: + # valid files the parser flags with syntax errors (tests/ly_corpus.rs). + name: LilyPond corpus + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Fetch LilyPond v2.26.0 (regression tests and snippets only) + run: | + git clone --depth 1 --branch v2.26.0 --filter=blob:none --sparse \ + https://gitlab.com/lilypond/lilypond.git "$RUNNER_TEMP/lilypond" + git -C "$RUNNER_TEMP/lilypond" sparse-checkout set input/regression Documentation/snippets + - run: cargo test --test ly_corpus -- --ignored --nocapture + env: + LYTK_LILYPOND_SRC: ${{ runner.temp }}/lilypond + # The LilyPond-shaped fuzz properties, with many more cases than in + # `cargo test` (release build: inputs at the reader's bounds are slow + # unoptimised). + - run: cargo test --release --test fuzz_inputs lilypond_shaped + env: + LYTK_FUZZ_CASES: "1024" + msrv: name: MSRV (rust-version in Cargo.toml) runs-on: ubuntu-latest @@ -78,8 +104,13 @@ jobs: - run: cargo check --all-targets python: - name: Python tests + name: Python ${{ matrix.python }} tests runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The abi3 floor (lilycorpus runs on 3.10) up to the newest. + python: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 with: @@ -88,7 +119,7 @@ jobs: - uses: Swatinem/rust-cache@v2 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python }} - name: Create and expose virtualenv run: | python -m venv .venv @@ -98,11 +129,14 @@ jobs: - run: python -m pip install --upgrade pip - run: pip install maturin pytest pytest-benchmark scipy # CPU-only torch so the torch dataset-adapter tests run instead of skipping - # (scipy above enables the eval-metric tests too). - - run: pip install torch --index-url https://download.pytorch.org/whl/cpu + # (scipy above enables the eval-metric tests too). On one Python only: + # the frameworks are large, and the others test the bindings. + - if: matrix.python == '3.12' + run: pip install torch --index-url https://download.pytorch.org/whl/cpu # TensorFlow too: its dtype rules are stricter than torch's, so the tf # data-loader path fails on inputs the torch path accepts. Without this # every tf test silently skips and that class of bug ships. - - run: pip install tensorflow-cpu + - if: matrix.python == '3.12' + run: pip install tensorflow-cpu - run: maturin develop - run: pytest tests/ -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a1f8a5..518a4ed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,6 +79,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + # A tag other than the version would publish a wheel under the wrong + # number (the version comes from Cargo.toml). + - name: Tag matches the version + if: startsWith(github.ref, 'refs/tags/v') + run: | + version=$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1) + if [ "${GITHUB_REF_NAME#v}" != "$version" ]; then + echo "::error::tag $GITHUB_REF_NAME does not match version $version in Cargo.toml" + exit 1 + fi - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test diff --git a/CLAUDE.md b/CLAUDE.md index 96b77a4..440c9df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -55,7 +55,7 @@ Six layers: - `TimeSignature.beats` is a `String` (supports compound like "3+2"), use `.beats_fraction()` for the `Frac` value 3. **Adapters** (`src/adapters/`) — format converters, all go through IR: - - `ly_to_ir/` — LilyPond parser using tree-sitter AST walk. Handles `\relative`, variables, `<< \\ >>` multi-voice, `\include`, figured bass, lyrics. The walk writes a positioned `Timeline` per part (`ly_to_ir/timeline.rs`); measures are made once, at score assembly + - `ly_to_ir/` — LilyPond parser using tree-sitter AST walk. Handles `\relative`, variables, `<< \\ >>` multi-voice, figured bass, lyrics; it does not follow `\include` (it warns: flatten first). The walk writes a positioned `Timeline` per part (`ly_to_ir/timeline.rs`); measures are made once, at score assembly. It reports `Diagnostic`s (`src/diagnostics.rs`): syntax errors from the tree (`ly_to_ir/syntax.rs`) and, from the walk's drop sites (`WalkState::report`), what it rejects or does not read. `builtins.rs` (LilyPond's commands, for `unknown-command`) is generated by `scripts/ly_builtins.py`. Strings, markup text and `\header` fields are decoded in `ly_to_ir/text.rs`, as LilyPond's lexer does - `mxml_to_ir/` — MusicXML reader using `musicxml` crate (typed struct traversal). Handles `.xml` and `.mxl` natively. - `ir_to_ly/` (~3400 lines) — IR to LilyPond emitter. Handles multi-staff piano scores, voice filtering, relative pitch mode - `ir_to_mxml/` — IR to MusicXML writer using `musicxml` crate (struct construction + serialization). Native MXL support. @@ -71,7 +71,7 @@ Six layers: 5. **ML representations** (`src/representations/`) — muspy-style encodings over the Layer-1 Music tree: `note_array` (onset/duration/pitch/velocity rows), `event_sequence` (Performance-RNN-style event codes), `piano_roll` (T × 128 matrix), `metrics`. All exposed to Python (`to_note_array`, `to_piano_roll`, `to_event_sequence` + inverses). Structured note navigation lives in `src/navigation.rs`. -6. **CLI** (`src/lytk/cli.py`) — the one `lytk` command, a Typer app over the Python bindings: `convert`, `transpose` (`-s`/`--interval`/`--to-key`), `invert`, `retrograde`, `change-language`, `abs2rel`, `rel2abs`, `info`, `positions`, `bundle`, `batch`, `diff`, `flatten`. Folder conversion and `batch` run in worker processes (`ProcessPoolExecutor`). There is no Rust binary: the crate is a library only. The Python package `src/lytk/` also ships datasets (`lytk.datasets`) and eval metrics (`lytk.metrics`). +6. **CLI** (`src/lytk/cli.py`) — the one `lytk` command, a Typer app over the Python bindings: `convert`, `transpose` (`-s`/`--interval`/`--to-key`), `invert`, `retrograde`, `change-language`, `abs2rel`, `rel2abs`, `info`, `positions`, `bundle`, `batch`, `diff`, `flatten`, `check`. Folder conversion and `batch` run in worker processes (`ProcessPoolExecutor`). There is no Rust binary: the crate is a library only. The Python package `src/lytk/` also ships datasets (`lytk.datasets`) and eval metrics (`lytk.metrics`). ## Key Design Patterns diff --git a/Cargo.lock b/Cargo.lock index 45ffa67..bac71e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,7 +457,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lytk" -version = "0.2.0" +version = "0.3.0" dependencies = [ "cc", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 5ede3b2..416cb76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lytk" -version = "0.2.0" +version = "0.3.0" edition = "2021" # Floor set by our oldest dependency (clap 4.6). Matters because the PyPI sdist # is compiled on the user's own toolchain. diff --git a/README.md b/README.md index 348ec2c..6fd51fc 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,20 @@ lytk.to_lilypond(score, "output.ly") # or to_musicxml, to_midi, to_abc, abc = lytk.to_abc(score) # without a path, writers return the text ``` +Check LilyPond before you rely on it: + +```python +for d in lytk.check_lilypond(text, semantic=True): # errors and warnings, in source order + print(d) # 3:12: error: missing `}` [missing-token] +score = lytk.from_lilypond("score.ly", strict=True) # raises lytk.LilyPondSyntaxError on an error +score.diagnostics # what the reading reported, either way +score.header # every \header field, as a dict +fields = lytk.header_fields(text) # …or read from the text: key, value, span +``` + +A reader raises `lytk.ParseError` (a `ValueError`) when its input cannot be +read, and `OSError` when the file cannot be opened. + Turn a score into arrays: ```python @@ -101,6 +115,7 @@ lytk convert input.xml -o output.ly # formats are taken from the exten lytk convert corpus/ -o out/ -f xml -j 8 # a whole folder, in parallel lytk transpose input.ly -s 3 -o up.ly lytk flatten score.ly -o flat.ly # inline every \include +lytk check score.ly --semantic # report the errors of LilyPond files lytk info input.mxl ``` @@ -126,9 +141,10 @@ lyrics); slurs and ornaments are not performed. ## Not there yet -- Tablature, percussion (unpitched notes) and fretboard diagrams. -- MIDI import transcribes the file as it is. It does not yet quantize, separate - voices, detect tuplets or split the hands of a piano part. +- Tablature, unpitched notes and fretboard diagrams (LilyPond drums are read + as their General MIDI keys on a percussion staff). +- Performed MIDI is quantized less reliably when its tempo is far from the + file's own, when chords are released unevenly, or at a low resolution. - Non-traditional key signatures, cross-staff notes (`\change Staff`), and Humdrum spine splits (`*^`, `*v`). Files that use spine splits are rejected with an error. diff --git a/SECURITY.md b/SECURITY.md index b376b0c..0fa80b3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,9 +19,10 @@ reachable from a malicious input file are in scope, in particular: - panics or unbounded memory/CPU use while parsing (a reachable denial-of-service for any service that converts user uploads) -- path traversal or unintended file reads — note that LilyPond `\include` is - followed by design, so callers that accept untrusted `.ly` files should run - conversion in a sandbox and pass only trusted `-I` search paths +- path traversal or unintended file reads. The LilyPond reader does not + follow `\include` (it reports it); `lytk flatten` / `lytk.flatten` does, by + design, so callers that flatten untrusted `.ly` files should run it in a + sandbox and pass only trusted `-I` search paths - decompression bombs in `.mxl` archives (these are read with a size cap) Release builds keep `overflow-checks` on, so an integer overflow panics rather diff --git a/docs/changelog.md b/docs/changelog.md index d899d8f..56542a7 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,191 @@ engineering notes are in the [development log](devlog.md). ## [Unreleased] +## [0.3.0] - 2026-09-27 + +LilyPond input you can trust. Readers raise `lytk.ParseError` instead of +crashing, `lytk.check_lilypond` and `lytk check` report diagnostics, and +`strict=True` rejects what LilyPond rejects. Strings and headers are read +as LilyPond reads them, pitch-language files are honoured, and top-level +music, `\book`, drum mode and `\fixed` follow LilyPond. The inputs that made +lytk panic, hang or run out of memory, found by fuzzing and a hunt, are +fixed, and all of LilyPond's 2,626 regression tests and snippets are read. + +### Added + +- `lytk.__version__`, the crate's version (`lytk --version` prints it). + +- `lytk.LytkError`, the base of the errors lytk raises, and + `lytk.InternalError`: a Rust panic inside any function or method that + reads, writes or transforms music now raises `InternalError` (an ordinary + `Exception`, carrying the panic's message and source location) and prints + nothing. It used to escape as pyo3's `PanicException`, a `BaseException` + that `except Exception` does not catch, with its message on stderr. +- `lytk.ParseError`, raised by every reader (`from_*`, `Score.from_json`, + `Score.from_dict`, `MusicDocument.from_json`, `flatten`) when its input + cannot be read. It subclasses both `LytkError` and `ValueError`, so + `except ValueError` keeps working. I/O failures stay `OSError`. +- Diagnostics for LilyPond: `lytk.check_lilypond(text, *, semantic=False)` + returns `lytk.Diagnostic` objects (severity, code, message, line, column + and a character span), from the parse tree alone by default (about 0.1 ms + per file), from a full reading with `semantic=True`. Errors are input + LilyPond rejects: syntax errors, unclosed brackets, a `>>` without its + `<<`, what is left of a Scheme expression missing its `(`, `c3`, a tuplet + or multiplier with a zero term, plain text. Warnings are input lytk reads + around: unknown commands, includes, unknown languages, dropped music, + `\midi`-only scores, values it cannot represent. The codes are listed in + `docs/import-export.md`. +- `strict=False` on every LilyPond reader: `strict=True` raises + `lytk.LilyPondSyntaxError` (a `ParseError`) when the reading has an error, + with every diagnostic in its `diagnostics` attribute. + `Score.diagnostics` and `MusicDocument.diagnostics` hold them either way + (they are not part of `to_dict`/`to_json`). +- `lytk check FILE… [--semantic] [--json]`: the diagnostics of LilyPond + files; exit status 1 on an error. +- Rust: `diagnostics::{Diagnostic, Severity}`; `LyToIrAdapter::read_str` and + `read_file` return a `LyReading` (every movement and the diagnostics); + `ly_to_ir::check` and `ly_to_ir::read_source`. +- `Score.header` and `MusicDocument.header`: every header field as a dict + (`title`, `subtitle`, `composer`, `arranger`, `lyricist`, then the others by + key), and `MusicDocument.lyricist`. +- `lytk.header_fields(text)`: every `\header` field of LilyPond text as a + `HeaderField` (key, value as text, character span of the whole `key = value`, + and the `\score` block it is in), from the parse tree alone, so a field can + be read and cut without reading the music. Rust: `ly_to_ir::header_fields`. +- LilyPond drum mode: `\drums { bd4 sn }`, `\drummode`, `\new DrumStaff` + and `\new DrumVoice` read LilyPond's 128 drum names as their General MIDI + keys (`bd` is key 36) on a percussion staff, which MIDI export puts on + channel 10. A word that is no drum name is `unrecognized-token`. +- LilyPond `\fixed c' { … }`: absolute pitches an octave up per mark of the + reference pitch, also inside `\relative` and in a variable. +- LilyPond `\book` and `\bookpart`: their scores and music are movements, + and their `\header` is the book's. + +### Changed + +- Each top-level LilyPond music expression is a movement, as LilyPond makes + a score of each: `{ c'1 } { d'1 }` is two movements, and music beside + `\score` blocks takes its place among them. It was dropped + (`dropped-music`), or merged into one movement. A music variable used at + the top level (`\m`) is read; it was dropped. Notes outside braces at the + top level (`c'4 d'`) are a `syntax-error`, as in LilyPond; they were a + warning. +- The shorthands `\chords { … }`, `\figures { … }` and `\lyrics { … }` are + chord names, figured bass and lyrics; their contents were read as notes. + +- The version has one source, `Cargo.toml` (`pyproject.toml` declares it + dynamic); the release workflow fails when the tag is not that version, and + CI runs the Python tests on 3.10 to 3.13. +- Readers raise `lytk.ParseError` where they raised `ValueError` (a subclass, + so existing handlers still catch it). A `.ly` file that is not UTF-8 is a + `ParseError`; it was an `OSError`. +- LilyPond markup is text, not music: `c4 -\markup \bold a8 d4` has two notes, + as LilyPond reads it (`a8` is the markup's word), and the words of a + top-level `\markup`, of `\tempo \markup …` or of a `\set`/`\override` + markup value no longer become notes. Top-level `\layout` and `\midi` blocks + are no longer read as music either. +- A UTF-8 byte-order mark is whitespace anywhere in LilyPond input, as in + LilyPond; the grammar reported one past the start as a syntax error. +- `\time 0/4` is ignored like `\time 1/0`, as LilyPond does. +- LilyPond strings are decoded as LilyPond reads them, in every context + (headers, lyrics, `\tempo`, markup, `\with`, `\mark`, `\set`, context and + voice names, `\language`): `\"`, `\\`, `\n`, `\t` and `\'`, any other + backslash kept. A string used to end at its first escape: + `texidoc = "Some \"doc\" here"` read as `Some `. +- A header value given as `\markup` is its plain text + (`composer = \markup { \bold "J. S." Bach }` is `J. S. Bach`), and `#"…"` + its string; both were dropped. Markup text at a note keeps its words + (`c4^\markup { \italic dolce }`), not only its strings. +- Headers are scoped: a `\score`'s `\header` belongs to that movement only (it + leaked into the next ones), and the top-level `\header` fills every + movement's unset fields, wherever it stands. +- The LilyPond writer writes every header field (it wrote none unless a + title, composer, arranger or poet was set), the others sorted by key and + quoted when they are no identifier; the Music path writes the poet and the + other fields too. Text directions at notes are written + (`^\markup { "dolce" }`); they were lost. The Music path quotes lyrics as + the Score path does. +- `\include "english.ly"` and LilyPond's other language files set the pitch + language, as `\language` does (`arabic.ly`: Italian names); the files of + pitch names lytk cannot read (`makam.ly`, `persian.ly`, `bagpipe.ly`, …) + raise `unknown-language`. `\language` is honoured inside `\score` and + music, not only at the top level, and an unknown name keeps the language in + force instead of resetting it to Dutch. +- The LilyPond reader refuses input past its bounds with a `ParseError` (a + `ValueError`: "… lytk refuses input this large") instead of hanging, running out of + memory or crashing: more than 500,000 notes, rests and chords once repeats + and variables are expanded; music longer than 100,000 whole notes or with + 100,000 bars; a duration multiplier above 16,777,216 (`R1*1000000000`); + music nested deeper than 2,000 levels; a pitch outside octaves -128..127; a + staff group of more than 255 staves. LilyPond's own 2,626 regression tests + and documentation snippets are all read. +- Constructs whose numbers the IR cannot hold are dropped, and the music + around them is read: a time signature with a 0 or 256+ denominator + (`\time 3/256` used to become 3/0), a tuplet with a 0 term or one above + 255 (its notes are read unscaled), a measure length with a 0 denominator, + a duration multiplier with a 0 or oversized denominator. +- A written duration that is not a power of two up to 1024 (`c3`, `c2048`) + reads as a quarter note, as LilyPond reads its "not a duration". +- `\repeat unfold` inside `\relative` repeats the same pitches, as LilyPond + does: each copy used to be read relative to the previous one, climbing. +- `Score.from_dict`, `Score.from_json` and `MusicDocument.from_json` refuse + values no reader makes, with a `ValueError`: a 0 time-signature + denominator or tuplet term, a negative or oversized duration, an octave + outside -128..127, an alteration beyond ±4, time-signature beats above + 10,000. +- `transpose` takes at most ±127 semitones, `invert` an axis in octaves + -128..127 with an alteration of at most ±4, and an interval number is + 1..99; other values are a `ValueError` (they overflowed in Rust). +- MusicXML export: when no `divisions` up to 65535 represents every duration + exactly (several coprime tuplets, e.g. 3, 5, 7, 9, 11 and 13 in different + voices), 10080 is used and other durations are rounded. The least common + multiple used to overflow, or be truncated to 16 bits and write wrong + durations. + +### Fixed + +- Inputs that panicked: `\time N/0`, `\time 3/256`, `\tuplet 0/N`, + `\times N/0`, `#(ly:make-moment N 0)`, 256 or more dots on a figured-bass + figure, a zero-length note (`c4*0`) in LilyPond export, 256 or more staves + in a PianoStaff or in `MusicDocument.to_score()`, coprime durations whose + positions overflowed, note arrays of long music. +- Inputs that hung or exhausted memory: `s1*N`, `R1*N` and `\skip 1*N` for + a huge N, nested `\repeat unfold`, variables that double one another, a + variable redefined in terms of itself and used inside `\relative`, a tiny + `measureLength`, an over-long note (bar lines were laid to its end and + thrown away), a relative passage climbing hundreds of octaves (every + writer wrote each octave mark out), and nesting of `\tuplet`, `\relative` + or `\repeat` a few hundred levels deep, which overflowed the stack. The + reader now walks on a thread with a stack of its own. +- `\ottava` with a number past what a shift is (`#2147483647`) panicked; it + is clamped to three octaves. `\ottava #2` read as size 16 instead of 15 (a + 15ma in MusicXML), and the Music-path LilyPond writer wrote an 8va as + `\ottava #8` and an 8vb as an 8va. +- A `**kern` note with 256 or more dots panicked. +- Music longer than 100,000 whole notes from MusicXML, ABC or `**kern` (one + MusicXML note of 25 million whole notes) was accepted, and the ABC writer + tied it over every bar line until an allocation aborted the process. The + three readers now refuse it, as the LilyPond and MIDI readers do, and the + ABC writer refuses music of more than 100,000 bars. +- Time that grew with the square of the input: meter changes (reading and + lowering; 100,000 notes with 40,000 meter changes took 24 s through every + writer, now 5 s), long runs of tuplets in ABC output, long runs of grace + notes in the Music-path LilyPond output, and many diagnostics on one long + line. +- A tremolo `:N` past 1024 (`c1:2147483648`) is not read as one: it gave + 31 tremolo marks, and the LilyPond writer's shift overflowed. The writers + cap the marks of hand-made IR at 10. +- `lytk.flatten` on a diamond of includes (a file including the next one + twice, level after level) doubled its output at every level until memory + ran out. It now stops with a `ValueError` after 10,000 includes or 64 MiB + of output. +- A LilyPond context whose music follows a nested `\new` or a mode + (`\new Staff \new Voice { … }` at the top level, + `\new Staff \drummode { … }`, `\new Staff \fixed c' { … }`) left its + music to be read as a separate expression. +- LilyPond part ids after a variable definition started at `P2`. +- The MusicXML writer panicked on figured bass at the beat of a grace note. + ## [0.2.0] - 2026-09-27 MIDI and ABC conversion rebuilt and checked against independent references @@ -270,9 +455,10 @@ First public release. - Readers and writers for LilyPond, MusicXML, compressed MusicXML (`.mxl`), MIDI, ABC and Humdrum `**kern`, all through one internal representation. -- A LilyPond reader for real files: variables, `\relative`, `\include`, piano - scores with several voices per staff, repeats and voltas, cadenzas, lyrics, - chord names and figured bass. The LilyPond writer supports all 12 +- A LilyPond reader for real files: variables, `\relative`, piano scores with + several voices per staff, repeats and voltas, cadenzas, lyrics, chord names + and figured bass. (It does not follow `\include`: `lytk flatten` inlines + includes first.) The LilyPond writer supports all 12 note-name languages. - Transforms: transpose (by semitones, interval or target key), invert, retrograde, and note-name language changes. @@ -291,6 +477,7 @@ First public release. newer. - MIT licence. -[Unreleased]: https://github.com/CSCPadova/lytk/compare/v0.2.0...HEAD +[Unreleased]: https://github.com/CSCPadova/lytk/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/CSCPadova/lytk/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/CSCPadova/lytk/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/CSCPadova/lytk/releases/tag/v0.1.0 diff --git a/docs/cli.md b/docs/cli.md index c4ab7ad..b6f0042 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -18,6 +18,7 @@ shows each one's options. | [`diff`](#diff) | Compare two scores by what they sound | | [`batch`](#batch) | Run a JSON list of conversion jobs | | [`flatten`](#flatten) | Inline every `\include` of a LilyPond file | +| [`check`](#check) | Report the errors in LilyPond files | ## Formats and streams @@ -235,6 +236,24 @@ warning on stderr. More than one `\header` block is an error. Each inlined file is wrapped in `% === BEGIN INCLUDE: … ===` / `% === END INCLUDE: … ===` comments; `--no-markers` leaves them out. +## check + +``` +lytk check FILE.ly ... [--semantic] [--json] +``` + +Checks LilyPond files (`-` reads stdin) and prints one line per finding, +`FILE:LINE:COLUMN: SEVERITY: MESSAGE [CODE]`, such as +`score.ly:12:7: error: `>>` without a matching `<<` [syntax-error]`. By +default only the syntax is checked, from the parse tree, in well under a +millisecond per file. `--semantic` reads the files too and adds what a reading +reports: invalid durations and ratios, commands neither LilyPond nor the file +defines, includes it does not follow, music it drops. The codes are listed in +[import-export.md](import-export.md#diagnostics). `--json` writes a list of +`{"file", "severity", "code", "message", "line", "column", "start", "end"}` +objects instead. The exit status is 1 when a file has an error; warnings alone +exit with 0. + ## From Python Everything above is also available as Python functions (`lytk.from_musicxml`, diff --git a/docs/design.md b/docs/design.md index e74f092..878c601 100644 --- a/docs/design.md +++ b/docs/design.md @@ -249,15 +249,19 @@ Implemented transforms: `Transpose`, `ChangeLanguage`, `Invert`, `Retrograde`. ## Testing Strategy -- **Unit tests** — colocated with each module (`#[cfg(test)]`). 255 passing. -- **Fixture tests** — 143 MusicXML files from `musicxmlTestSuite/` in `tests/fixtures/xml/`. -- **Round-trip tests** — `LilyPond → IR → LilyPond` and `MusicXML → IR → MusicXML`, - comparing semantic equivalence. Currently tested via `test_roundtrip_*` in the adapter tests. -- **CLI integration tests** — `tests/cli.rs` using `assert_cmd` + `predicates` + `tempfile`. - 15 passing. -- **Python tests** — `pytest tests/` (26 tests) for PyO3 binding correctness. -- **Criterion benchmarks** — `benches/benchmarks.rs`, 25 benchmarks. -- **pytest-benchmark** — `benches/bench_python.py`, cross-language comparison. +- **Unit tests** — colocated with each module (`#[cfg(test)]`). +- **Integration tests** (`tests/*.rs`) — the MusicXML test suite and LilyPond + fixtures (`tests/fixtures/`), round trips per format with a fidelity + scoreboard (`fidelity.rs`), LilyPond-shaped fuzzing (`fuzz_inputs.rs`), the + LilyPond corpus boards (`ly_corpus.rs`, CI job `lilypond-corpus`), the + LilyPond oracle (`lilypond-oracle` job) and the diagnostics and text tests. +- **Python tests** (`tests/*.py`) — the bindings, the CLI (Typer's + `CliRunner`), datasets, representations and end-to-end conversions. +- **Benchmarks** — Criterion (`benches/benchmarks.rs`) and pytest-benchmark + (`benches/bench_python.py`), cross-language. + +Counts change with every release; `docs/roadmap.md` and the devlog give the +current ones. --- diff --git a/docs/development.md b/docs/development.md index 1227b48..51642db 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,7 +4,7 @@ | Tool | Version | Purpose | |---|---|---| -| Rust + Cargo | ≥ 1.85 | Compile the Rust library and CLI (`rust-version` in `Cargo.toml`) | +| Rust + Cargo | ≥ 1.85 | Compile the Rust library (`rust-version` in `Cargo.toml`) | | Python | ≥ 3.10 | Python bindings and tooling | | uv | latest | Python dependency management | | maturin | ≥ 1.0, < 2.0 | Build the PyO3 extension | @@ -26,7 +26,7 @@ uv tool install maturin ## Compiling -### Rust — library and CLI +### Rust — library Debug build (fast compile, no optimisations): diff --git a/docs/devlog.md b/docs/devlog.md index 84eef52..77b4bd5 100644 --- a/docs/devlog.md +++ b/docs/devlog.md @@ -3,6 +3,409 @@ Dated engineering notes, newest first. The release history is in [changelog.md](changelog.md). +## 2026-09-27 — Release 0.3.0 + +Epic J (J0–J7) ships as **0.3.0**. The version was already 0.3.0 in +`Cargo.toml` since J6, its only source; `[Unreleased]` became +`[0.3.0] - 2026-09-27` in the changelog. As for 0.2.0, the release commit +reaches `master` through a pull request and the tag goes on the merge +commit; the release workflow's test gate checks that the tag is the +version. Next: Epic K (0.4.0), and lilycorpus can move to `lytk>=0.3,<0.4` +and drop its workarounds (docs/lytk-wishlist.md there). + +## 2026-09-27 — Epic J: reading problems the hunt left open (J7) + +The hunt's "modes" area read phantom notes: `\drums { bd4 sn }` as two +unrecognized words, and the contents of `\chords`, `\figures` and `\lyrics` +as notes. Top-level music followed rules of its own, not LilyPond's. Checked +against LilyPond 2.22 before changing anything: + +- each top-level music expression is a score of its own (`{ } { }` makes two; + so does `\new Staff { } \new Staff { }`), and music beside `\score` blocks + is one more, in its place; +- a music variable at the top level (`\m`) is read as music; +- notes outside braces at the top level are a syntax error + ("unexpected NOTENAME_PITCH"); +- `\book { \score { } \score { } }` makes two scores; +- `\chords { }` and `\drums { }` each make one. + +The reader now follows these. `walk_top` walks the top level and the inside +of `\book`/`\bookpart` alike; each top-level expression opens a movement +(`begin_movement`) and closes the one before, `\score` closes the open one, +and `\addlyrics` stays with its expression. The movement state (parts, +positions, lyrics, chord names) is reset once, in `finish_movement`, instead +of being saved and restored around each `\score`. `dropped-music` is left +with one job: the movements a single-movement reader drops. + +Drum mode reads LilyPond's drum names as General MIDI keys. The table +(`ly_to_ir/drums.rs`, 128 names) is generated by `scripts/ly_builtins.py` +from `ly/drumpitch-init.ly` (`drumPitchNames` → `midiDrumPitches`); `tamtam` +has no MIDI key and is left out. `\drums` opens a `DrumStaff` with a +percussion clef, whose part MIDI export puts on channel 10. `\fixed c' { }` +reads absolute pitches offset by the reference's marks, also inside +`\relative` and in a variable. + +A context's music after a nested `\new` or a mode (`\new Staff \new Voice +{ }`, `\new Staff \drummode { }`) used to be left to the next top-level +step, which worked only because that step read it into the same score; with +a movement per expression it would have split the staff from its music. +Context bodies now go through `walk_body`, which reads nested contexts and +mode prefixes. Part ids no longer skip `P1` after a variable definition (the +capture of a variable counted its parts). + +CI's fuzz job then found a panic the new figure placement exposed, not +caused: the MusicXML writer, at a grace note (no time) right after writing +the figures due at its beat, asked for the figures of an empty range, which +`BTreeMap::range` refuses. `\new FiguredBass` with a grace note at a +figure's beat hit it before. + +Board (d) on LilyPond 2.26.0 (count/files): `dropped-music` 0 (was 77/65), +`unrecognized-token` 247/42 (was 437/78); the others unchanged, board (a) +still 1 file, no file refused. + +Tests: 1,143 Rust (`tests/ly_modes.rs`), 229 Python. + +## 2026-09-27 — Epic J: J1's hunt, finished + +The five areas the first hunt left (commands, values, modes, resources, +other readers) went through a scratch harness, one process at a time under +`ulimit -v 4000000`: 308 hand-made cases, each read by every reader it fits, +checked (syntax and semantic), and sent through every writer, transform, +representation and the JSON round trip, with panics caught and time and peak +memory measured per stage. + +Found and fixed: +- *Panics*: `\ottava #2147483647` (`n * 8` overflowed) and + `#-2147483648` (`abs`); a `**kern` note with 300 dots (the dot counter). + The ottava sizes were also inconsistent: the LilyPond reader wrote `n * 8` + (16 for a 15ma), the Score-path writer divided by 8 (a MusicXML 15 came out + as `\ottava #1`), and the Music-path writer wrote the size itself + (`\ottava #8` for an 8va) without its sign. `OctaveShift::from_octaves` and + `octaves` now keep MusicXML's 8, 15, 22 throughout. +- *An abort*: a MusicXML note of 25 million whole notes read fine, and the + ABC writer (which bars the Music tree by the meter) tied it over every bar + line until a 1.6 GB allocation failed; an abort, which no firewall catches. + The MusicXML, ABC and `**kern` readers now refuse music past 100,000 whole + notes (`adapters::check_*_length`, an f64 estimate that cannot overflow), as + the LilyPond and MIDI readers did, and the ABC writer, fallible now, refuses + more than 100,000 bars (hand-made IR has no reader bound). +- *Quadratic time*, measured by doubling inputs of 18 repeated constructs + (a 4× input must take about 4× the time): + - meter changes: `Grid::build` retained over every meter at each change + (reading and lowering; 20,000 repetitions of two meters: 5.8 s → 0.6 s); + the cadenza lookup per bar is a sweep now too; + - the ABC writer counted a tuplet run to its end at every group; + - the Music-path LilyPond writer copied the grace group at every grace; + - a diagnostic counted its column from its line's start + (`diagnostics::Columns` counts on along the line). + `repeated_constructs_scale_linearly` pins the four with a time ratio, which + does not depend on the machine's speed. + +Not bugs, recorded here: at the reader's bound (450,000 notes) every stage +takes about 20 s in all, and the MusicXML writer about 4 KB of memory per +note (the `musicxml` crate serializes through a DOM of its own), 2 GB at the +bound. The two panics and the abort were the whole harvest of the hunt's +308 cases; the other readers held (zero divisions, SMPTE time, lying chunk +lengths, huge counts), within linear time. + +Tests: 1,138 Rust, 229 Python. + +## 2026-09-27 — Epic J: J6 (release hygiene) + +The version is `Cargo.toml`'s alone: `pyproject.toml` declares it dynamic +(maturin reads it), `_core` exports it as `__version__`, and +`test_version_is_the_crates` checks both against the file. It is 0.3.0 now, +the release Epic J is for. The release workflow's test gate fails when the +tag is not `v` + that version. The Python CI job runs on 3.10, 3.11, 3.12 and +3.13; torch and TensorFlow, large, are installed on 3.12 only, so the others +test the bindings and the CLI. + +Docs: SECURITY.md said the reader follows `\include` (it reports it; only +`flatten` follows includes), and so did the 0.1.0 changelog entry; README's +"not there yet" described the MIDI importer before 0.2.0; `design.md` gave +test counts from the first weeks and a Rust CLI test suite that no longer +exists; `development.md` spoke of a Rust CLI. `src/tree-sitter/README.md` +records the vendored grammar's upstream commit, found by matching blob hashes +against upstream's history: `b3b38a6` (Tree-sitter 0.26.2); the grammar is +unchanged upstream since (only regenerated, with 0.27.0). + +## 2026-09-27 — Epic J: J4 (strings and headers) and J5 (pitch language) + +**J4.** `ly_to_ir/text.rs` decodes text as LilyPond's lexer does +(`Lily_lexer::escaped_char`): a string joins its fragments and decodes `\n`, +`\t`, `\\`, `\"`, `\'`, keeping any other backslash; `#"…"` decodes Guile's +common escapes; markup becomes its words and strings, one space where the +source has anything between them (`\markup { \bold "J. S." Bach }` is +`J. S. Bach`). Every reader site goes through `extract_string_value`, so the +fifteen callers the plan listed were fixed at once; it used to return the +first fragment (`texidoc = "Some \"doc\" here"` read as `Some `). + +Headers: +- `header_block_fields` reads a `\header` block (string, `\markup` and `#"…"` + values; `##f` and other Scheme left out), for both the walk and + `header_fields`. +- Scoping: a `\score`'s `\header` writes the movement's metadata, which is + taken before the block and restored after it; top-level `\header` fields go + to `book_header` and fill every movement's unset fields at the end, last + assignment first, so a book header after the scores still applies, as in + LilyPond. +- `header_fields(text)` walks the tree once, remembering the `\score` blocks it + passes to give each field its score index. In Python its spans are + character offsets, like `Diagnostic`'s (the plan said bytes; Python slices + strings by character). + +Writers: one `helpers::header_block` for both paths writes every field (the +Score path wrote none without a title, composer, arranger or poet; the Music +path left out the poet and the others), the others sorted by key and quoted +when they are no LilyPond identifier. The round-trip test found note text +directions lost by the Score-path writer: `attachments_to_ly` writes them now. +The Music path's lyric quoting is the Score path's. + +Acceptance: `snippet_headers` (in the `lilypond-corpus` CI job) compares the +`texidoc` and `categories` of the 389 snippets with a decoder that finds them +by text search: 778 values equal, 108 of the texidocs with escapes. The string +round trip (`tests/ly_text.rs`) puts a quote, a backslash, a newline and +non-ASCII in headers, `\tempo`, `\mark`, instrument names, lyrics and markup +text, and reads the written LilyPond back unchanged. + +**J5.** `WalkState::include` maps LilyPond's language files to their language +(`arabic.ly`, Italian names with accidentals of its own, to Italian); the +five files of pitch names lytk cannot read (`bagpipe`, `hel-arabic`, `makam`, +`persian`, `turkish-makam`) raise `unknown-language`, any other file +`ignored-include`. `set_language` serves `\language` at the top level, in +`\score` and in music; an unknown name keeps the language in force. A music +variable keeps the language of its definition for the re-reading inside +`\relative`. + +Board (d) after both (LilyPond 2.26.0, count/files): `ignored-include` +221/171 (was 245/192), `unknown-language` 6/6 (new: the files above), +`unrecognized-token` 437/78 (was 595/89); the others unchanged. Board (a) +still 1 file. + +Tests: 1,133 Rust, 228 Python (`TestStringsAndHeaders`). + +**Next**: J6 (release hygiene), then 0.3.0. J1's unfinished hunt, later. The +reading gaps the warnings show: a music variable at the top level, `\fixed`, +drum mode, and the headers of `\book` blocks (not read). + +## 2026-09-27 — Epic J: J2 (exception hierarchy) and J3 (diagnostics, strict mode) + +**J2.** `ParseError(LytkError, ValueError)` and `LilyPondSyntaxError(ParseError)` +need two bases, which `create_exception!` cannot give: they are made with +`type()` at module init and kept in `GILOnceCell`s. Readers map through +`read_err` (I/O stays `OSError`, the rest is `ParseError`); writers keep +`ValueError`. `ly_to_ir::read_source` makes a non-UTF-8 `.ly` a parse error. +`Diagnostic` has `__reduce__`, so a `LilyPondSyntaxError` survives pickling +(lilycorpus validates in worker processes). + +**J3.** `src/diagnostics.rs` holds `Diagnostic` (byte range, 1-based line, +column in characters; Python gets character offsets, converted in one pass). +Two sources: +- *Syntax* (`ly_to_ir/syntax.rs`, tree only): ERROR nodes (`missing `}`` at + the opener when the ERROR holds an unclosed bracket or quote, else + `unexpected …`), MISSING nodes, and three checks the grammar cannot make: a + `>` outside a chord that is no accent (a `>>` without `<<`); the leftovers + of a Scheme expression that lost an opening `(` (a `.` after a Scheme value, + a parenthesis after one unless it is a command's argument, a number, string + or quote on the line of a top-level Scheme expression that starts it); a + markup `\override` given a quoted atom instead of a pair. Markup is skipped + (any word or mark is text there). +- *Semantic* (`WalkState::report`, at the walk's drop sites): the codes in + `docs/import-export.md`. `unknown-command` checks `builtins.rs`, LilyPond + 2.26's 1,347 commands generated by `scripts/ly_builtins.py` (keywords, the + `ly/` identifiers, markup commands, context names, units), and the names the + file assigns (`name = …`, `"\\|" = …`, `#(define …)`). A chord-mode block + that the walk reads a second time as notes is walked `quiet`. Diagnostics are + sorted and deduplicated (music inside `\relative` is walked twice). + +Precision first: an error must mean LilyPond rejects the input, so valid files +stay readable in strict mode. Calibration on the corpus removed the false +alarms of a first draft: bare words and numbers at the top level are only +judged at the start of the file (elsewhere they may be a function's +arguments, `\testStyle PianoStaff { … } teaching`); `\time 1/0` is a warning +(LilyPond 2.26 only warns, `time-denominator-zero.ly`); `\vshape #'(…) (` and +Mutopia's tagline (`\hspace #-1.0 .`) led to the command-argument and markup +exceptions. + +Boards (LilyPond v2.26.0, 2,626 files): +- (a) syntax errors: 1 file, `other/display-lily-tests.ly` (`##[ … #]`, which + the grammar does not know); the BOM of `bom-mark.ly` is whitespace now. The + reading adds no error: 1 file. `check_lilypond` median 0.077 ms (release). +- (b) mutations: 499 of 502 caught, all by the syntax check. The board's + "Scheme parenthesis" used to include slurs, whose deletion leaves valid + LilyPond; it now takes `scheme_list` parentheses only (42 each way). The 3 + missed leave valid LilyPond: `#'(4 4)` without its `(` is `#'4` and the note + `4)`. +- (d) warnings (count, files): `dropped-music` 77/65, `ignored-include` + 245/192, `skipped-score` 52/51, `unknown-command` 188/45 (mostly commands of + included files), `unrecognized-token` 595/89 (drum mode, other pitch + languages set by an include, chord and lyric text read as notes), + `unsupported-value` 2/2. + +Reading changes. Markup is skipped as text: `c4 -\markup \bold a8 d4` read the +note a8, where LilyPond reads the word `a8` (checked with LilyPond 2.22: its +bar check fails); top-level `\markup`, `\tempo \markup`, `\set`/`\override` +markup values and top-level `\layout`/`\midi` blocks no longer feed the music +walker. Two PDMX fixtures have that `\markup \bold a8`: their first bar is +now short and does not survive the LY round trip, so the fidelity board's LY +bars went 27 → 26 (`tests/fidelity.rs`). + +Tests: 1,125 Rust (`tests/ly_diagnostics.rs`: each code on its input, and +valid constructs that must raise nothing), 224 Python (`TestExceptionHierarchy`, +`TestDiagnostics`, `TestCheck`). The LilyPond-shaped fuzzer found one more +panic on the way: `c1:2147483648` read as 31 tremolo marks, which the LilyPond +writer shifted past 32 bits. `:N` past 1024 is no tremolo now, and the writers +cap hand-made IR at 10 marks (`oversized_tremolos_are_written_capped`). + +A lesson in the process: building the old HEAD in a worktree with a shared +`CARGO_TARGET_DIR` made later builds in the main tree link the worktree's +code (two fidelity runs "passed" on the old reader); `cargo clean -p lytk` +fixed it. Build other trees with their own target directory. + +**Next** (reading gaps the warnings now show; not J3): +- A music variable used at the top level (`\music`) is dropped, as is music + given to commands the walk does not know (`\fixed c' { … }`, `\drummode`): + `dropped-music` / `unrecognized-token` say so. +- J1's unfinished hunt (commands, values, modes, resources, other readers). +- J4 (strings and headers), J5 (language files: the `\include "english.ly"` + behind many `unrecognized-token`), J6. + +## 2026-09-27 — Epic J: J0 (corpus boards) and J1 (no panic reaches Python) + +**J0.** `tests/ly_corpus.rs` holds three boards: +- `syntax_board` (ignored; CI job `lilypond-corpus` sparse-clones LilyPond + v2.26.0): 2 of 2,626 valid files come out with ERROR nodes (`bom-mark.ly`, + `other/display-lily-tests.ly`), 0.3–0.5 ms a file in a debug build. +- `reader_board` (same job): every valid file read, none refused by J1's + bounds. Its first run caught a real bug (below): 16 regression tests were + refused for climbing past octave 20, then 4 past 127. +- `mutation_board` (every `cargo test`): one token deleted from each + committed fixture. Braces, string quotes, chord `>` and `>>` are flagged + 100 %. Deleting `<<` (4/46) or a Scheme `(` (4/47) almost never is: the + lone `>>` or `)` left behind parses as `punctuation`, and a lone `)` or + `>` is legal LilyPond (a slur end, the `->` accent). Only the reader's + context can tell (J3). + +Board (d), dropped input per diagnostic code, waits for J3's diagnostics. + +**J1, the hunt.** A read-only multi-agent workflow probed the built +extension, one subprocess per input under `ulimit -v`. It ran two hunters to +the end (IR assembly and downstream, structure) before it was stopped. The +five unfinished ones (commands, values, modes, resources, other readers) +are follow-ups. With six hunters probing in parallel at up to 6 GB each, +plus an unbounded fuzz run, the machine ran out of memory. Probes and heavy +tests now run one at a time under `ulimit -v 4000000`. + +**J1, the fixes.** +- *Firewall* (`src/python.rs`): every binding that reads, writes or + transforms runs in `guard()`, which catches the unwind and raises + `lytk.InternalError` (base `lytk.LytkError`, both in module `lytk`, so they + pickle). A panic hook installed at module init records the message and + location instead of printing them while any guard is active. The counter + is global, not per thread, because the LilyPond reader walks on a thread + of its own. `_panic_for_tests` checks the firewall from Python. +- *Root causes*: + - `parse_fraction` and `parse_ly_make_moment` refuse 0 denominators. + - `\time` drops denominators past `u8`. + - `\tuplet`/`\times` read their music unscaled when a term is 0 or past `u8`. + - Figured-bass dots saturate. + - `beats_fraction` and `actual_duration` are total. + - `lilypond_log` handles zero-length notes. + - Staff groups of 256+ staves are refused; the lowering's staff and voice + numbers saturate. +- *Bounds* (`ly_to_ir/state.rs`); past one the reading stops with a + `ValueError`: + - 500,000 generated elements, spent on every push, unfold pass and + variable splice; + - 100,000 whole notes, checked with `checked_add`, so coprime + denominators refuse instead of overflowing; + - multiplier terms of at most 2^24; + - durations that are powers of two up to 1024; + - octaves -128..127; + - walk depth 2,000, re-walks of variables included. +- *Grid* (`src/ir/timeline.rs`): + - `MAX_BARS` = 100,000, and the reader refuses a grid that reaches it. + - Unless notes are tied on, bar lines stop at the bar holding the last + start; `c'1*1000000000` used to cut a billion bars and then drop them. +- *Stack*: the reader runs on a 64 MB thread. Nested `\tuplet` overflowed + 8 MB at about 400 levels, because `handle_escaped_word`'s frame is about + 20 KB in debug builds. +- *Variables*: + - Inside `\relative`, a variable being read again is not read again + inside itself, and a block that names its own variable uses its + captured music. `a = { c4 } a = { \a \a \a }` then + `\relative { \a }` was 3^16 notes and is now 3. + - Splices count against the budget, so twenty-two doubling definitions + are refused in under a second. +- *Semantics fix*: `\repeat unfold` inside `\relative` restores the + reference pitch at every pass. LilyPond relativizes the written body once + and copies it; lytk read each pass relative to the last, and LilyPond's + page-layout tests climbed to octave 128. +- *Hand-supplied IR*: `from_dict` and `from_json` check divisors, + durations, octaves, alterations and beats before deserializing. +- *Arguments*: `transpose` ±127, `invert` axis bounds, interval numbers + 1..99. +- *Writers*: MusicXML `divisions` use a checked lcm and fall back to 10080 + when the value exceeds `u16`. `frac_to_steps` computes in `i128`. + +**Memory.** The fuzz properties were the out-of-memory trigger: +- `to_piano_roll` caps at 16M steps, which is 2 GiB. +- The downstream property now builds rolls at resolution 4 and skips scores + over 5,000 bars. +- Refused inputs peak at about 310 MB in a debug build, at the + 500,000-element bound. +- `cargo test` peaks at 560 MB. +- `LYTK_FUZZ_CASES` sets the case count: 32/16 locally, 1,024 in the CI job + (release build). The reader property takes about 180 s in debug for 256 + cases. + +**Next**: +- Finish the hunt for the five unfinished areas, one probe at a time under + a memory limit. +- ~~`lytk.flatten` on a diamond of includes (`f(i)` including `f(i+1)` + twice) expands 2^depth.~~ Done: it stops after 10,000 includes or 64 MiB + of output (`FlattenError::TooLarge`). +- Then J2 (the exception hierarchy on top of `LytkError`) and J3 + (diagnostics: the dropped constructs above become warnings or errors). + +## 2026-09-27 — Release plan 0.3.0 → 0.5.0 (planning, no code) + +The roadmap gains Epics J (0.3.0), K (0.4.0) and L (0.5.0), from +lilycorpus's request list (`docs/lytk-wishlist.md` there) checked against +0.2.0: nothing on it changed in 0.2.0, which touched MIDI and ABC only. + +**Grammar measurement** (the J0 baseline; no in-repo harness yet). The +grammar was compiled on its own, `gcc -shared -fPIC -O1 -I +src/tree-sitter/src src/tree-sitter/src/parser.c -o liblilypond.so` (ABI +14, no external scanner), loaded through `ctypes` into py-tree-sitter 0.25 +(`Language(lib.tree_sitter_lilypond())`), and every file was checked for +ERROR and MISSING nodes: +- `lilypond/input/regression/*.ly` (v2.27.3): 1 of 2,155, `bom-mark.ly`: a + UTF-8 BOM that LilyPond treats as whitespace. +- Its subdirectories: 1 of 162, `other/display-lily-tests.ly` (`##[ #]`). +- `Documentation/snippets`: 0 of 389; `tests/fixtures/ly`: 0 of 35. +- 0.34 s for the 2,155 files. + +Hand-made broken inputs: +- ERROR/MISSING for an unclosed `{`, an extra `}`, an unclosed string, + chord (`MISSING >`) and `<< >>` (`MISSING >>`), Scheme (`MISSING )`), + block comment, `\score` brace. +- Nothing for `\foobarbaz`, `c3`, `\time 0/0`, a stray `@` and plain words: + the grammar is token-level (`{ c4 @ d4 }` is `expression_block (symbol) + (unsigned_integer) (punctuation) (symbol) (unsigned_integer)`), so those + are for the walk to report (J3). + +**Found while planning** (details in the J tasks): +- Six LilyPond inputs panic through the bindings. +- `extract_string_value` keeps only a string's first fragment, in fifteen + places. +- `\include "english.ly"` is ignored, so `cs4` is dropped. +- The reader never follows `\include`, although CLAUDE.md, + `import-export.md` and SECURITY.md say it does. +- Header metadata leaks from one `\score` into the next. +- `FolderDataset`'s cache is keyed by path, never invalidated, and bypassed + by `Subset`. + ## 2026-09-27 — Release 0.2.0 Epic I (with P9, performed MIDI) ships as **0.2.0**: version bumped in diff --git a/docs/import-export.md b/docs/import-export.md index 0e5a1a1..3d7f9b3 100644 --- a/docs/import-export.md +++ b/docs/import-export.md @@ -177,25 +177,61 @@ Layer 1 (`MusicDocument`, the Music tree the ML representations consume). | Multi-voice | ✅ | `<< \\\\ >>` syntax | | Multi-staff | ✅ | `\new Staff`, `\new PianoStaff` | | Variables | ✅ | Definition + resolution | -| `\include` | ✅ | File inlining | +| Movements | ✅ | Each `\score` block and each top-level music expression (`{ }`, `<< >>`, `\new Staff`, `\relative`, a music variable), in order, as LilyPond makes a score of each; `\book`/`\bookpart` scores and music too, their `\header` the book's | +| `\include` | 🟡 | LilyPond's language files (`english.ly`, …) set the pitch language; other files are not followed (`ignored-include`: flatten first) | | `\transpose` | ✅ | Transposing instrument context | | Relative mode | ✅ | `\relative` pitch context | +| `\fixed` | ✅ | Absolute pitches an octave up per mark of the reference, also inside `\relative` | +| Drum mode | ✅ | `\drums`, `\drummode`, `\new DrumStaff`/`DrumVoice`: LilyPond's 128 drum names as General MIDI keys (`bd` = 36) on a percussion staff (MIDI channel 10) | | Glissando | ✅ | `\glissando` with style override | | Arpeggio | ✅ | `\arpeggio` with direction | | Slide | ✅ | Via glissando trill style | | Anacrusis | ✅ | `\partial` | | Paper block | ✅ | `\paper { }` → page layout | +| `\header` | ✅ | Every field (`Score.header`); strings decoded as LilyPond does; `\markup` values as plain text, `#"…"` as the string; a `\score`'s header is its movement's, the top-level one every movement's | +| `\language` | ✅ | At the top level, in `\score` and in music; an unknown name keeps the language in force | | `\set Staff.instrumentName` | ✅ | Part name from `\set` property | | `\set Staff.midiInstrument` | ✅ | MIDI instrument from `\set` property | | `\context Voice = "name"` | ✅ | Named voices for lyrics attachment | -| Lyrics | ✅ | `\lyricsto`, `\lyricmode`, `\context Lyrics`, `\addlyrics` | +| Lyrics | ✅ | `\lyricsto`, `\lyricmode`, `\lyrics`, `\context Lyrics`, `\addlyrics` | | Staff variables | ✅ | `staffX = \new Staff { ... }` with full part metadata | | `\cadenzaOn/Off` | ✅ | Free-time span collapses to one `senza_misura` measure (both hands, score-wide) | | `\melisma/End` | ✅ | Gracefully skipped | | `\autoBeamOff/On` | ✅ | Gracefully skipped | | `\dynamicUp/Down` | ✅ | Gracefully skipped | -| Harmony/chord names | ✅ | `\chordmode` (root, quality, bass; language-aware) | -| Figured bass | ✅ | `\figuremode` (figures, accidentals incl. natural & double) | +| Harmony/chord names | ✅ | `\chordmode`, `\chords` (root, quality, bass; language-aware) | +| Figured bass | ✅ | `\figuremode`, `\figures` (figures, accidentals incl. natural & double) | + +### Diagnostics + +The reader reports what it finds wrong and what it does not read +(`LyReading.diagnostics` in Rust; `Score.diagnostics`, +`lytk.check_lilypond` and `lytk check` in Python). An error is input LilyPond +itself rejects: `strict=True` readers raise `lytk.LilyPondSyntaxError` on one. +A warning is input lytk reads around. + +| Code | Severity | Raised by | +|---|---|---| +| `syntax-error` | error | What the grammar cannot parse; notes outside braces at the top level (`c'4 d'`); a `>>` without its `<<`; what is left of a Scheme expression that lost an opening `(` (`#set-global-staff-size 20)`); a markup `\override` whose argument is not a pair | +| `missing-token` | error | A bracket or quote never closed (`{ c'4 d'4`, `<< … }`) | +| `invalid-duration` | error | A duration that is not a power of two (`c3`); read as a quarter | +| `invalid-ratio` | error | A zero term in a tuplet (`\tuplet 0/2`), a duration multiplier (`*1/0`) or a measure length | +| `not-lilypond` | error | Words at the start of the file that are no LilyPond (plain text) | +| `too-large` | error | Input past the reader's bounds (`check_lilypond` only; a reader raises `ParseError`) | +| `unknown-command` | warning | A command neither LilyPond 2.26 nor the file defines (a typo, or one from an include) | +| `unrecognized-token` | warning | A word in music that is no note name in the current language, or no drum name in drum mode | +| `ignored-include` | warning | `\include`: the reader does not follow includes; flatten the file first | +| `unknown-language` | warning | `\language` with a name lytk does not know | +| `dropped-music` | warning | The movements after the first, from a reader that takes one (`from_lilypond`, `convert_str`) | +| `skipped-score` | warning | A `\score` with `\midi` and no `\layout` | +| `unsupported-value` | warning | A value LilyPond accepts but lytk cannot represent (durations past 1024, time-signature denominators or tuplet terms past 255), or ignores as LilyPond does (`\time 1/0`) | + +`check_lilypond(text)` checks the syntax only, from the parse tree (about +0.1 ms per file on LilyPond's regression tests); `semantic=True` reads the text +too. On LilyPond 2.26's 2,626 regression tests and snippets, one file has an +error (`display-lily-tests.ly`, whose `##[ … #]` the grammar does not know); +of 502 fixtures broken by deleting one brace, quote, `<<`, `>>`, chord `>` or +Scheme parenthesis, 499 have one. ### Export (IR → LilyPond) — `src/adapters/ir_to_ly.rs` diff --git a/docs/roadmap.md b/docs/roadmap.md index fbacb23..b1da735 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,7 +2,59 @@ Items are grouped by status. Completed items are kept for reference. -## Latest status (2026-09-24, third pass) +## Latest status (2026-09-27, Epic J: J6, the hunt, J7) + +**Epic J is done and released as v0.3.0.** The version has one source and +the Python tests run on 3.10–3.13 (J6). The hunt's last five areas found two +panics, an abort and four quadratic paths, all fixed. The LilyPond reader +now reads top-level music as LilyPond does, a movement per expression, and +`\book`, drum mode, `\fixed` and the `\chords`/`\figures`/`\lyrics` +shorthands (J7). Test counts: 1,143 Rust + 229 Python. Details in the +devlog. + +--- + +## Previous status (2026-09-27, Epic J: J4 and J5) + +**LilyPond text reads as LilyPond reads it.** Strings are decoded in every +context (a string used to stop at its first escape), header values given as +`\markup` or `#"…"` become text, each `\score`'s header stays with its +movement, and `Score.header` / `lytk.header_fields` expose every field (the +latter with its span, to cut it). `\include "english.ly"` and the other +language files set the pitch language, and `\language` works inside +`\score` and music. The writers write every header field and the text at +notes. Test counts: 1,133 Rust + 228 Python. Details in the devlog. + +--- + +## Previous status (2026-09-27, Epic J: J2 and J3) + +**LilyPond reading reports what is wrong.** Readers raise `lytk.ParseError` +(a `ValueError`) on input they cannot read. `lytk.check_lilypond` and +`lytk check` report `Diagnostic`s: syntax errors from the tree alone (0.08 ms +median per file), and with `semantic=True` what the walk rejects or does not +read. `strict=True` readers raise `LilyPondSyntaxError`, and every LilyPond +`Score`/`MusicDocument` carries its diagnostics. On LilyPond 2.26's 2,626 +files one has an error (a grammar gap); of 502 broken fixtures, 499 have one. +Markup is no longer read as music. Test counts: 1,125 Rust + 224 Python. +Details in the devlog. + +--- + +## Previous status (2026-09-27, Epic J: J0 and J1) + +**No panic reaches Python; LilyPond input cannot hang lytk or exhaust its +memory.** Every binding runs inside a panic firewall (`lytk.InternalError`), +the LilyPond reader has bounds (elements, length, bars, multipliers, nesting, +octaves) and walks on its own 64 MB stack, and a multi-agent hunt's findings +are fixed and pinned in `tests/fuzz_inputs.rs` together with LilyPond-shaped +fuzzing. `tests/ly_corpus.rs` measures the grammar and the reader against +LilyPond's own 2,626 regression tests and snippets (CI job `lilypond-corpus`). +Test counts: 1,114 Rust + 214 Python. Details in the devlog. + +--- + +## Previous status (2026-09-24, third pass) **One CLI, in Python.** The Rust binary (`src/main.rs`, clap) is gone; the `lytk` command is a Typer app (`src/lytk/cli.py`) with all 13 subcommands of @@ -593,6 +645,96 @@ Follow-ups the oracles found outside IA1/IB (LilyPond writer and reader): --- +## Release plan 0.3.0 → 0.5.0 — LilyPond input you can trust (planned ⬜) + +Planned 2026-09-27. Three releases that make lytk a dependable reader of +LilyPond written by people and by models. They come out of +[lilycorpus](https://github.com/matteospanio/lilycorpus), the LilyPond +training corpus built on lytk (its `docs/lytk-wishlist.md` is the request +list), and out of LLM evaluation, where the question "is this LilyPond +well-formed, and where is it wrong?" needs an answer in milliseconds instead +of a `lilypond` run. + +Today the reader never fails and never reports: +- `""`, `"}}} INVALID {{{"` and `"{ c4 d e f"` all return a score; +- every walker drops the nodes it does not recognise without a trace; +- strings are cut at their first escape; +- `\include "english.ly"` loses notes; +- six input families panic. + +**Measured 2026-09-27** (lytk's grammar compiled from `src/tree-sitter/src/parser.c` and loaded with py-tree-sitter 0.25; LilyPond sources at `v2.27.3`, the reference checkout in `lilypond/`): + +| Corpus (valid LilyPond) | Files | With ERROR/MISSING nodes | Cause | +|---|---|---|---| +| `input/regression/*.ly` | 2,155 | 1 | a byte-order mark mid-file (`bom-mark.ly`) | +| `input/regression/*/*.ly` | 162 | 1 | `other/display-lily-tests.ly` (`##[ #]`) | +| `Documentation/snippets/*.ly` | 389 | 0 | | +| `tests/fixtures/ly/*.ly` | 35 | 0 | | + +- Parsing all 2,155 top-level regression files takes 0.34 s, 0.16 ms a file. +- **Caught by the tree:** unclosed braces, strings, chords (`MISSING >`), `<< >>` (`MISSING >>`), Scheme parentheses and block comments. +- **Not caught:** unknown commands, `c3`, `\time 0/0`, stray tokens and plain words. The grammar is nearly token-level (`c4` is `symbol` + `unsigned_integer`), so these can only be found by the walk, where they are dropped today. + +So strict mode can rest on the tree for syntax, and the walk must report +what it drops. Principles, as in Epic I: +- measure first, gate on committed baselines that may only improve; +- keep the Python API backward compatible within the series: new keywords + default to today's behaviour, and new exceptions subclass the ones raised + today. + +### Epic J — 0.3.0: trustworthy LilyPond reading (✅ released in v0.3.0, 2026-09-27) + +| Task | Description | Acceptance | Size | Status | +|------|-------------|------------|------|--------| +| J0 | **Measure.** Add `tests/ly_corpus.rs`, marked `#[ignore]`, run by a new CI job that sparse-clones LilyPond at a pinned tag (`input/regression`, `Documentation/snippets`), like `lilypond-oracle`. It holds four boards: (a) error diagnostics on valid files, (b) detection of structural mutations of the fixtures (a brace, quote, `>`/`>>` or `)` deleted; truncation), (c) panics under LilyPond-shaped fuzz, (d) dropped-input counts per diagnostic code on valid files, to calibrate J3's warnings | Boards committed with the numbers above as baselines | M | ✅ 2026-09-27: boards (a), (b) and a reader board (valid files refused: 0 of 2,626); (c) is J1's fuzz; (d), the warnings per code, came with J3 | +| J1 | **No panic reaches Python.** (1) Fix the six known families at their source, each becoming a diagnostic: `\time N/0` (`consume.rs` `parse_fraction`); `\time 3/256` (`den as u8`, `music.rs:404`); `\tuplet 0/N` and `\times N/0` (`music.rs:503-509`); `ly:make-moment N 0` (`consume.rs:794-815`); figured-bass dot overflow (`figured_bass.rs:191`). (2) Make the shared sinks total: `TimeSignature::beats_fraction` (`measure.rs:122`) and `Duration::actual_duration` (`duration.rs:85`); reject `beat_type == 0` and `tuplet_actual == 0` in `Score.from_dict`/`from_json`. (3) Bound expansions: `s*N` and `\skip` loops, figured-bass `*N`, nested `\repeat unfold`, with a total-events limit like MIDI's 500,000 bars. (4) Firewall: one `guard()` helper around every `#[pyfunction]` that reads, writes or transforms, turning a panic into `lytk.InternalError` with its message; a panic hook installed at module init keeps guarded panics off stderr (pattern: `mxml_to_ir::catch_read`, `tests/common` `safe()`). (5) Fuzz: a LilyPond-shaped proptest strategy in `tests/fuzz_inputs.rs` (edge numbers 0/1/255/256/`u32::MAX` in `\time`, `\tuplet`, `\times`, `*N`, dots, nesting) over `convert_str`, `_multi`, `_to_music`, `with_language` | Six families return diagnostics; fuzz board 0 panics; no `PanicException` from the bindings (Python tests on the six inputs) | M | ✅ 2026-09-27 (the six families are dropped constructs until J3 reports them); the hunt's five unfinished areas were finished after J6: two panics (`\ottava`, `**kern` dots), an abort (over-long music in the ABC writer), four quadratic paths; see the devlog | +| J2 | **Exception hierarchy.** `lytk.LytkError(Exception)`; `ParseError(LytkError, ValueError)`; `LilyPondSyntaxError(ParseError)` carrying `.diagnostics`; `InternalError(LytkError)` for caught panics; I/O stays `OSError`. A non-UTF-8 `.ly` becomes a `ParseError` (today `OSError`: changelog *Changed*). Stubs list what each function raises | `except ValueError` code keeps working; `tests/test_bindings.py` covers each class | S | ✅ 2026-09-27: every reader, `from_json`/`from_dict` and `flatten`; `ParseError`/`LilyPondSyntaxError` are made with `type()` at module init (two bases); `Diagnostic` pickles, so the error crosses process pools | +| J3 | **Diagnostics and strict mode.** New `src/diagnostics.rs`: `Diagnostic { severity, code, message, line, column, start, end }`, collected in `WalkState`. *Syntax:* ERROR and MISSING nodes, at the empty `has_error()` block in `ly_to_ir/mod.rs:203-205`, as `syntax-error` / `missing-token` ("missing `>>`"). *Semantic errors:* `invalid-duration` (`c3`), `invalid-ratio` (from J1), bare words at top level (`not-lilypond`). *Warnings, where the walk drops input today:* `unknown-command`, `unrecognized-token`, `ignored-include`, `unknown-language`, `dropped-music` (top-level music beside a `\score`, movements a single-score reader drops), `skipped-score` (`\midi` without `\layout`), `expansion-limit`. Ignored Scheme is normal LilyPond: no warning. The drop sites are the catch-all arms in `walk.rs` (135-137, 350, 515, 677, 836, 850, 1132, 1209, 1336, 1339) and `music.rs` (139-145, 875-888, 910). A UTF-8 BOM anywhere is whitespace, as in LilyPond. *Python:* `lytk.check_lilypond(text, *, semantic=False) -> list[Diagnostic]` (syntax only by default, no IR); `strict=False` on every `from_lilypond*` reader (`True` raises `LilyPondSyntaxError` on any error); `Score.diagnostics` and `MusicDocument.diagnostics` (kept out of `to_dict`/`to_json`); `Diagnostic.__str__` → `3:12: error: missing '}' [missing-token]`. *CLI:* `lytk check FILE… [--json] [--semantic]`, exit 1 on errors | Board (a) ≤ 1 error on the valid corpus after the BOM fix; board (b) ≥ 99 % of structural mutations detected; the semantic cases above detected; `check_lilypond` median ≤ 1 ms on the regression files; board (d) published in the devlog | L | ✅ 2026-09-27: (a) 1 file (`##[ #]`), in the reading too; (b) 499/502, all from the syntax check (a slur's parenthesis no longer counts as a Scheme one); median 0.077 ms. Changed from the plan: `expansion-limit` is not needed (past a bound the reading fails, and `check_lilypond` reports `too-large`); `unsupported-value` for values lytk cannot hold; `\time 1/0` is a warning, as in LilyPond 2.26. Three syntax checks beyond the tree: a stray `>>`, the leftovers of a Scheme `(` deleted, a markup `\override` without a pair. `unknown-command` knows LilyPond 2.26's 1,347 commands (`scripts/ly_builtins.py`) | +| J4 | **Strings and headers.** `extract_string_value` (`consume.rs:442-451`) joins every `string_fragment` and decodes each `escape_sequence` as LilyPond's lexer does (`lily/lexer.ll`: `\n` `\t` `\\` `\'` `\"`; any other backslash is kept). One fix for all fifteen callers: headers, lyrics, `\tempo`, markup, `\with`, `\mark`, `\clef`, `\bar`, `\set`, `instrumentName`, `\lyricsto`, `\context = "…"`, `\language`. Header values given as `\markup` become their plain text, and `#"…"` becomes a string. Headers are scoped: a `\score`'s `\header` no longer leaks into the next movement, and a top-level `\header` applies to every movement. *Python:* `Score.header` / `MusicDocument.header` → `dict[str, str]` of every field, plus `MusicDocument.lyricist`; `lytk.header_fields(text) -> list[HeaderField]` (key, value, byte span, score index) from the tree, no IR, so a caller can read a field and cut it. *Writers:* the Score path writes `\header` whenever any field is set (today only for title/composer/arranger/lyricist, `ir_to_ly/mod.rs:156-159`); the Music path writes lyricist and `extra` (`music_emit.rs:129-151`); `extra` keys are written sorted, not in `HashMap` order | String round trip in every context with `\"`, `\\`, newline and non-ASCII; `texidoc` and `categories` of all 389 official snippets equal to an independent escape-aware decoder | M | ✅ 2026-09-27: `ly_to_ir/text.rs` decodes strings, `#"…"` and markup text; the 389 snippets' `texidoc` and `categories` (778 values, 108 with escapes) equal the oracle (`snippet_headers`); the string round trip covers headers, `\tempo`, `\mark`, instrument names, lyrics and markup text (`tests/ly_text.rs`). `HeaderField` spans are character offsets in Python, as `Diagnostic`'s. Also: the Score-path writer now writes note text directions (the round trip found them lost) | +| J5 | **Pitch language.** `\include ".ly"` for LilyPond's language files sets the language as `\language` does: the 11 lytk supports; `arabic.ly` → italiano; `bagpipe`, `makam`, `persian`, `turkish-makam` → `unknown-language` warning. `\language` is honoured inside `\score` and music (today top level only, `walk.rs:36-46`). An unknown name keeps the current language and warns (today it resets to Dutch, `walk.rs:41-42`) | `\include "english.ly" { cs4 d4 }` reads C♯4 D4 (today D4 only) | S | ✅ 2026-09-27: `hel-arabic.ly` joins the unreadable ones; a variable read again inside `\relative` keeps the language of its definition. On LilyPond 2.26's corpus `unrecognized-token` fell from 595 to 437 and `ignored-include` from 245 to 221 | +| J6 | **Release hygiene.** `lytk.__version__` (`env!("CARGO_PKG_VERSION")` exported by `_core`); one version source (pyproject `dynamic = ["version"]`, or a test that `Cargo.toml` and `pyproject.toml` agree); Python CI on 3.10–3.13 (only 3.12 today; lilycorpus supports 3.10); the release workflow fails when the tag differs from the version. Docs that say the reader follows `\include` (changelog 0.1.0, SECURITY.md; CLAUDE.md done in J3, `docs/import-export.md` in J5) are corrected; so are README's pre-0.2.0 MIDI limitations, `design.md` test counts, `development.md`'s Rust CLI, `python.rs`'s "python feature" and T11.2's batch `catch_unwind` (deleted with `src/main.rs`). The upstream commit of tree-sitter-lilypond is recorded in `src/tree-sitter/README.md` | CI green on four Pythons; `import lytk; lytk.__version__ == "0.3.0"` | S | ✅ 2026-09-27: version 0.3.0 from `Cargo.toml` alone (`dynamic = ["version"]`, `uv.lock` records it dynamic), `test_version_is_the_crates`; the Python job runs on 3.10–3.13 (torch and TensorFlow on 3.12 only); the vendored grammar is upstream `b3b38a6`, whose grammar is unchanged since; `python.rs` had lost its "python feature" mention with J1 | +| J7 | **Reading problems the hunt left open.** Each top-level music expression is a movement, as LilyPond makes a score of each (music beside `\score` was dropped, `\m` at the top level too); notes outside braces at the top level are a `syntax-error`; `\book`/`\bookpart` scores are movements; drum mode reads drum names; `\fixed`; `\chords`/`\figures`/`\lyrics` are no notes | Each rule checked against LilyPond 2.22; board (a) unchanged; `tests/ly_modes.rs` | M | ✅ 2026-09-27: `dropped-music` on the corpus 77 → 0, `unrecognized-token` 437 → 247; board (a) 1 file, 0 refused | + +Order: J0 → J1 + J2 → J3 → J4 + J5 → J6 → J7. + +### Epic K — 0.4.0: source-level API + +| Task | Description | Size | Status | +|------|-------------|------|--------| +| K1 | **`\version`.** Keep it in `ScoreMetadata` and expose it as `Score.lilypond_version`. `lytk.lilypond_version(text)` reads it from the tree (a commented-out one does not count). `set_lilypond_version(text, v)` and `strip_lilypond_version(text)` edit by span. A `LilyPondVersion` type compares numerically (`2.24` == `2.24.0`). `to_lilypond(…, version=)` and `to_lilypond_music(…, version=)` expose `IrToLyAdapter::with_version`: Python always gets `\version "2.24.0"` today (`python.rs:398, 421`). No convert-ly wrapper | S | ⬜ | +| K2 | **Includes.** Bind `flatten_string(text, *, base_dir=None, include_paths=(), add_markers=True)` (Rust `flatten_str` exists, `ly_flatten.rs:100-107`). Add `include_paths=` on the readers: given, they flatten first and report unresolved includes as `ignored-include`; not given, includes stay unfollowed, as SECURITY.md should say. Find includes on the tree instead of line by line: this catches includes not at the start of a line and fixes the one-line `%{ %}` that leaves flatten "inside a comment" and the false `MultipleHeaders` for a top-level plus a per-score `\header` | M | ⬜ | +| K3 | **Tokens.** `lytk.tokenize(text) -> list[Token]` (kind: comment, string, scheme, command, symbol, number, fraction, punctuation; text; byte span; line; column) from the tree's leaves; `lytk.strip_comments(text)`. Scheme can be sub-tokenized with the compiled but unused `LANGUAGE_LILYPOND_SCHEME` | S | ⬜ | +| K4 | **Statistics.** `lytk.info(score) -> dict` becomes the library home of `lytk info --json` (moved from `cli.py:484-529`; the CLI calls it), plus voices, total bars, duration in quarters, lyrics, chord symbols and grace notes. `lytk.source_stats(text)` from tokens: comments, Scheme expressions, bytes, tokens | S | ⬜ | +| K5 | **Movements.** Single-score readers report what they drop (`dropped-music`, from J3); add `from_lilypond_music_movements` for Layer-1 users (the datasets) | S | ⬜ | + +### Epic L — 0.5.0: datasets for curated corpora + +| Task | Description | Size | Status | +|------|-------------|------|--------| +| L1 | **`FolderDataset` robustness.** `on_error="raise" \| "skip" \| "warn"` with `dataset.errors` (today the first bad file ends every iteration). Cache keyed by file content + lytk version + normalized representation kwargs, written atomically (today keyed by path, never invalidated; `to_note_arrays()` and `iter_representation("note_array")` write different files). `Subset` delegates `_convert_item` to its parent: today it skips the cache, so the README's `split()` → dataloader example never uses it. `movements="first" \| "all"`. Pass-through of `language`, `include_paths`, `strict` and MIDI `quantize` | M | ⬜ | +| L2 | **Records dataset.** `RecordsDataset.from_jsonl(path, *, text_field="text", id_field="id", format="lilypond", split_field=None)` and `from_records(iterable)`. Items keep their id and metadata; `split(field=…)` returns the records' own splits (a deterministic, decontaminated split must not be re-shuffled by ratio). `Dataset.split(…, groups=)` for group-aware ratio splits | M | ⬜ | +| L3 | **Identity.** `FolderDataset.ids` (paths relative to the root); `return_ids=True` on the torch and tf adapters | S | ⬜ | +| L4 | **Python API reference.** `docs/python-api.md` generated from or checked against the stubs: lytk has none today (README + `_core.pyi`). It states that `from_lilypond_string` and `from_lilypond_music_string` share one parse (the Music tree is lifted from the Score) | S | ⬜ | + +### After 0.5.0 + +The pre-1.0 queue resumes: **P5** (enumerated notation types, the one +epic that breaks the Python API) as 0.6.0, then P3 → P4 → P6 → P8 → 1.0.0. +Decision for the owner: J–L go before P5 because they unblock lilycorpus +and are backward compatible, whereas the 0.1.0 plan put P5 first. P3's +import-options object should absorb J3/K2's keywords one to one: +`LilyPondReadOptions(language, strict, include_paths)`. + +Not planned: +- Semantic validation equal to LilyPond's. User-defined commands can come + from files lytk never reads, so compiling stays the authority. Strict + mode means "well-formed, and nothing lytk had to drop". +- A convert-ly wrapper: callers already run convert-ly. +- Making numpy and typer optional: `import lytk` imports neither already, + and moving typer to an extra would break `pip install lytk && lytk …`. + +--- + ## Humdrum (`**kern`) support — 2026-07-10 ✅ New `humdrum_to_ir`/`ir_to_humdrum` adapter pair (Layer-1, ABC-shaped), all @@ -757,7 +899,7 @@ notation-exported MIDI (IC). | Task | Description | Status | |------|-------------|--------| | T11.1 | Batch-job JSON schema `{in/input, out/output, format?, from?, transpose?, interval?}` | ✅ | -| T11.2 | `batch` executor (reuses rayon pool + per-job `catch_unwind` + non-zero exit on any failure) | ✅ | +| T11.2 | `batch` executor (reuses rayon pool + per-job `catch_unwind` + non-zero exit on any failure) | ✅; since the Python CLI (2026-09-24) jobs run in worker processes, and a panic is `lytk.InternalError` (J1) | | T11.3 | Per-job transform options (transpose/interval) ✅; visible-parts filter + filename templates ⬜ (overlap excerpt-selection, deferred) | 🟡 | | T11.4 | Diagnostic sidecars: `--report ` JSON of per-job ok/error | ✅ | @@ -787,6 +929,10 @@ notation-exported MIDI (IC). | **11** | **P1–P12** | **Pre-1.0.0 expansion (MuseScore-comparison backlog): CLI/IO + transforms, MusicXML fidelity, IR modeling, MIDI reconstruction — see plan file** | | **12** | **0.1.0** | **Preview release before the remaining P-epics; then P5 → P3 → P4 → P6 → P8 → P9** | | **13** | **I → 0.2.0** | **MIDI and ABC conversion repair (Epic I, with P9 performed MIDI): tagged `v0.2.0` 2026-09-27** | +| **14** | **J → 0.3.0** | **Trustworthy LilyPond reading: no panics, diagnostics + strict mode, whole strings and headers, pitch-language files: tagged `v0.3.0` 2026-09-27** | +| 15 | K → 0.4.0 | Source-level API: `\version`, includes from strings, tokens, statistics, movements | +| 16 | L → 0.5.0 | Datasets for curated corpora: robust `FolderDataset`, records dataset, ids, Python API reference | +| 17 | P5 → 0.6.0, then P3 → P4 → P6 → P8 → 1.0.0 | Pre-1.0 queue resumes (P5 breaks the Python API) | ## Key Decisions diff --git a/pyproject.toml b/pyproject.toml index 8dd434b..79a9fc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "lytk" -version = "0.2.0" +# The version is Cargo.toml's (maturin reads it): one source for both. +dynamic = ["version"] description = "Fast music-notation conversion and augmentation: LilyPond, MusicXML/MXL, MIDI, and ABC through a shared IR, with ML representations." readme = "README.md" authors = [ diff --git a/scripts/ly_builtins.py b/scripts/ly_builtins.py new file mode 100644 index 0000000..8c7c96a --- /dev/null +++ b/scripts/ly_builtins.py @@ -0,0 +1,102 @@ +"""Write src/adapters/ly_to_ir/builtins.rs, every command LilyPond defines, +and drums.rs, its drum-mode note names. + +The reader warns about a command it neither handles nor finds defined in the +file (`unknown-command`); the commands LilyPond itself defines are not +unknown. Run from the repository root, with a LilyPond git checkout: + + python3 scripts/ly_builtins.py lilypond v2.26.0 +""" + +import re +import subprocess +import sys + + +def show(repo: str, tag: str, path: str) -> str: + return subprocess.run( + ["git", "-C", repo, "show", f"{tag}:{path}"], + check=True, capture_output=True, text=True, errors="replace", + ).stdout + + +def files(repo: str, tag: str, folder: str, suffix: str) -> list[str]: + out = subprocess.run( + ["git", "-C", repo, "ls-tree", "--name-only", f"{tag}:{folder}"], + check=True, capture_output=True, text=True, + ).stdout + return [f"{folder}/{name}" for name in out.split() if name.endswith(suffix)] + + +def write_drums(repo: str, tag: str) -> None: + """src/adapters/ly_to_ir/drums.rs: drum-mode names and the MIDI key each + sounds, from `ly/drumpitch-init.ly` (`drumPitchNames`, `midiDrumPitches`).""" + text = show(repo, tag, "ly/drumpitch-init.ly") + names_part, pitches_part = text.split("midiDrumPitches")[0], text.split("midiDrumPitches")[1] + names = dict(re.findall(r"\((\w+) \. (\w+)\)", names_part)) + alters = {"NATURAL": 0, "SHARP": 1, "FLAT": -1, "DOUBLE-SHARP": 2, "DOUBLE-FLAT": -2} + pitches = {} + for sym, octave, step, alter in re.findall( + r"\((\w+) \. ,\(ly:make-pitch (-?\d+) (\d+) ([A-Z-]+)\)\)", pitches_part + ): + pitches[sym] = (int(octave), int(step), alters[alter]) + rows = [] + for name in sorted(names): + if names[name] not in pitches: + continue # no MIDI sound (`tamtam`) + o, st, al = pitches[names[name]] + # LilyPond's octave 0 holds middle C, lytk's octave 4. + rows.append(f' ("{name}", {o + 4}, {st}, {al}),') + with open("src/adapters/ly_to_ir/drums.rs", "w") as out: + out.write( + f"//! Drum-mode note names of LilyPond {tag.lstrip('v')} and the pitch whose MIDI\n" + "//! key sounds each (General MIDI percussion): `ly/drumpitch-init.ly`. Generated:\n" + f"//! `python3 scripts/ly_builtins.py lilypond {tag}`.\n\n" + "/// (name, octave, step 0-6 from C, alteration in semitones), sorted by name.\n" + f"pub(super) const DRUMS: &[(&str, i32, u8, i32)] = &[\n" + "\n".join(rows) + "\n];\n" + ) + print(len(rows), "drum names") + + +def main(repo: str, tag: str) -> None: + names: set[str] = set() + # Identifiers assigned at the top level of the init files: music + # functions, articulations, dynamics, property shorthands, spanners; + # `"\\<" = …` defines `\<` (and `"f" = …` the dynamic). The paper defaults (indented, inside + # `\paper`) hold the units, `\mm` and `\cm`; `\name Staff` defines the + # context `\Staff` of `\layout` blocks. + for path in files(repo, tag, "ly", ".ly"): + indent = r"\s*" if path.endswith("paper-defaults-init.ly") else "" + for line in show(repo, tag, path).splitlines(): + if m := re.match(indent + r"([A-Za-z](?:[-_]?[A-Za-z])*)\s*=", line): + names.add(m[1]) + elif m := re.match(r'"((?:[^"\\\\]|\\\\.)+)"\s*=', line): + name = m[1].replace("\\\\", "\\") + names.add(name.removeprefix("\\")) + elif m := re.match(r'\s*\\name "?([A-Za-z]\w*)', line): + names.add(m[1]) + # Markup commands (`\bold`, `\column`, …) and accordion registers. + for path in files(repo, tag, "scm", ".scm"): + text = show(repo, tag, path) + names.update(re.findall(r"\(define-markup(?:-list)?-command\s*\(([-\w]+)", text)) + names.update(re.findall(r"\(define-register-set\s+([-\w]+)", text)) + # The lexer's keywords and commands. + names.update(re.findall(r'\{"(\w+)", [A-Z_]+\}', show(repo, tag, "lily/lily-lexer.cc"))) + names.update(re.findall(r'\\\\(include|version|maininput|sourcefileline|sourcefilename)', + show(repo, tag, "lily/lexer.ll"))) + write_drums(repo, tag) + names = sorted(names) + rows = "\n".join(f' "{n.replace(chr(92), chr(92) * 2)}",' for n in names) + with open("src/adapters/ly_to_ir/builtins.rs", "w") as out: + out.write( + f"//! The commands LilyPond {tag.lstrip('v')} defines, without their backslash:\n" + "//! lexer keywords, the identifiers of its `ly/` init files and its markup\n" + f"//! commands ({len(names)}). Generated: `python3 scripts/ly_builtins.py lilypond {tag}`.\n\n" + "/// Sorted, for binary search.\n" + f"pub(super) const BUILTINS: &[&str] = &[\n{rows}\n];\n" + ) + print(len(names), "commands") + + +if __name__ == "__main__": + main(*sys.argv[1:3]) diff --git a/src/adapters/abc_to_ir.rs b/src/adapters/abc_to_ir.rs index ecc6c0e..41d89b7 100644 --- a/src/adapters/abc_to_ir.rs +++ b/src/adapters/abc_to_ir.rs @@ -82,7 +82,9 @@ impl ToMusicAdapter for AbcToIrAdapter { } fn convert_str_to_music(&self, text: &str) -> Result { - parse_tune(text) + let doc = parse_tune(text)?; + super::check_music_length(&doc.music)?; + Ok(doc) } } diff --git a/src/adapters/humdrum_to_ir.rs b/src/adapters/humdrum_to_ir.rs index 6ab3173..d0abe8c 100644 --- a/src/adapters/humdrum_to_ir.rs +++ b/src/adapters/humdrum_to_ir.rs @@ -44,7 +44,9 @@ impl ToMusicAdapter for HumdrumToIrAdapter { } fn convert_str_to_music(&self, text: &str) -> Result { - parse_kern(text) + let doc = parse_kern(text)?; + super::check_music_length(&doc.music)?; + Ok(doc) } } @@ -481,7 +483,7 @@ fn split_subtoken(sub: &str) -> (Duration, TokenFlags, String) { } } '%' => rational = Some((std::mem::take(&mut digits), String::new())), - '.' => dots += 1, + '.' => dots = dots.saturating_add(1), 'a'..='g' | 'A'..='G' | '#' | '-' | 'n' | 'r' => body.push(c), _ => {} // beams (L/J), ornaments, editorial marks — ignored } diff --git a/src/adapters/ir_to_abc.rs b/src/adapters/ir_to_abc.rs index 3c225e5..489521f 100644 --- a/src/adapters/ir_to_abc.rs +++ b/src/adapters/ir_to_abc.rs @@ -42,17 +42,22 @@ impl IrToAbcAdapter { impl FromMusicAdapter for IrToAbcAdapter { fn convert_music(&self, doc: &MusicDocument) -> Result { - Ok(emit_tune(doc)) + emit_tune(doc) } } +/// Most bar lines a voice is written with. A long note is tied over every +/// bar line of the meter, so a note of millions of whole notes (hand-made IR) +/// would be millions of tokens. +const MAX_BARS: usize = crate::ir::timeline::MAX_BARS; + /// One emitted ABC voice: an optional name and its flat event stream. struct OutVoice { name: Option, events: Vec, } -fn emit_tune(doc: &MusicDocument) -> String { +fn emit_tune(doc: &MusicDocument) -> Result { // Split the tree into top-level voices (parts / staves). A single voice keeps // the original single-line ABC; ≥2 voices emit `V:` blocks (ABC 2.1 §4.1). let voices: Vec = top_level_voices(&doc.music) @@ -130,7 +135,7 @@ fn emit_tune(doc: &MusicDocument) -> String { first_tempo.as_ref(), init_bar, pickup, - ); + )?; // No blank line: it would end the tune for every ABC reader. out.push_str(body.trim_end()); out.push('\n'); @@ -151,13 +156,13 @@ fn emit_tune(doc: &MusicDocument) -> String { first_tempo.as_ref(), init_bar, pickup, - ); + )?; out.push_str(body.trim_end()); out.push('\n'); } } } - out + Ok(out) } /// Emit one voice's body as a space-joined token string. The voice's first @@ -169,7 +174,7 @@ fn emit_body( header_tempo: Option<&TempoDirection>, init_bar: Option, mut pickup: Option, -) -> String { +) -> Result { let mut tokens: Vec = Vec::new(); let mut time_used = false; let mut tempo_used = false; @@ -189,6 +194,7 @@ fn emit_body( let mut this_bar: Option = None; let mut filled = Frac::new(0, 1); let mut bars_on_line = 0usize; + let mut bars = 0usize; // Sounding events still inside the open tuplet run (0 = not in a tuplet). let mut tuplet_left = 0usize; let mut queue: std::collections::VecDeque = events.iter().cloned().collect(); @@ -196,6 +202,11 @@ fn emit_body( // §7.4) just before its bar line. let mut layers: Vec> = Vec::new(); while let Some(mut ev) = queue.pop_front() { + if bars > MAX_BARS { + return Err(crate::adapters::AdapterError::Unsupported(format!( + "the music has more than {MAX_BARS} bars: not written as ABC" + ))); + } // A pickup: the first bar holds only its length. if sounding_duration(&ev).is_some() || matches!(ev, Music::Simultaneous(_)) { if let (Some(p), Some(len)) = (pickup.take(), bar_len) { @@ -256,11 +267,13 @@ fn emit_body( if tuplet_left == 0 { // One group per `p` notes: always fits inside a bar, so a // run never straddles a barline or a wrapped line. + // (`take` first: a long run of groups is not counted + // to its end at every group.) let run = std::iter::once(&ev) .chain(queue.iter()) + .take(ratio.0.max(1) as usize) .take_while(|m| tuplet_ratio(m) == Some(ratio)) - .count() - .min(ratio.0.max(1) as usize); + .count(); tokens.push(format!("({}:{}:{}", ratio.0, ratio.1, run)); tuplet_left = run; } @@ -344,6 +357,7 @@ fn emit_body( } else { tokens.push(tok); bars_on_line += 1; + bars += 1; } acc.bar(); filled = Frac::new(0, 1); @@ -365,6 +379,7 @@ fn emit_body( filled = Frac::new(0, 1); this_bar = None; bars_on_line += 1; + bars += 1; } } } @@ -382,7 +397,7 @@ fn emit_body( tokens.extend(words); } // Join on spaces, but keep the line breaks we inserted as real newlines. - tokens.join(" ").replace(" \n ", "\n").replace(" \n", "\n") + Ok(tokens.join(" ").replace(" \n ", "\n").replace(" \n", "\n")) } /// Write the bar's other voices, each after a `&`, with the accidentals the @@ -397,9 +412,9 @@ fn push_layers(tokens: &mut Vec, layers: &mut Vec>, acc: &mut if left == 0 { let run = layer[k..] .iter() + .take(r.0.max(1) as usize) .take_while(|x| tuplet_ratio(x) == Some(r)) - .count() - .min(r.0.max(1) as usize); + .count(); tokens.push(format!("({}:{}:{}", r.0, r.1, run)); left = run; } diff --git a/src/adapters/ir_to_ly/emit.rs b/src/adapters/ir_to_ly/emit.rs index f726614..bd4c460 100644 --- a/src/adapters/ir_to_ly/emit.rs +++ b/src/adapters/ir_to_ly/emit.rs @@ -220,11 +220,8 @@ pub(super) fn emit_measures( } } if let Some(oct) = &dir.octave_shift { - match oct.shift_type.as_str() { - "up" => parts.push(format!("\\ottava #{}", oct.size / 8)), - "down" => parts.push(format!("\\ottava #-{}", oct.size / 8)), - "stop" => parts.push("\\ottava #0".to_string()), - _ => {} + if matches!(oct.shift_type.as_str(), "up" | "down" | "stop") { + parts.push(format!("\\ottava #{}", oct.octaves())); } } if !parts.is_empty() { @@ -687,7 +684,10 @@ fn two_note_tremolo_to_ly( mode: PitchMode, prev: Option<&Pitch>, ) -> Option<(String, Pitch)> { - let marks = n1.tremolo_marks.max(n2.tremolo_marks); + let marks = n1 + .tremolo_marks + .max(n2.tremolo_marks) + .min(crate::ir::note::MAX_TREMOLO_MARKS); if marks == 0 { return None; } @@ -909,6 +909,17 @@ pub(super) fn attachments_to_ly(note: &Note) -> String { } } + // Text written at the note (`^\markup { "dolce" }`) + for td in ¬e.text_directions { + let dir = match td.placement { + crate::ir::articulation::Placement::Above => '^', + crate::ir::articulation::Placement::Below => '_', + _ => '-', + }; + let text = super::helpers::escape_ly_string(&td.text); + owned.push(format!("{dir}\\markup {{ \"{text}\" }}")); + } + let mut result: String = parts.join(""); for o in &owned { result.push_str(o); diff --git a/src/adapters/ir_to_ly/helpers.rs b/src/adapters/ir_to_ly/helpers.rs index c206d3a..af785f1 100644 --- a/src/adapters/ir_to_ly/helpers.rs +++ b/src/adapters/ir_to_ly/helpers.rs @@ -8,6 +8,45 @@ pub(super) fn escape_ly_string(text: &str) -> String { text.replace('\\', "\\\\").replace('"', "\\\"") } +/// The `\header { … }` block of `meta`, if it has a field: the fields the IR +/// names, then the others sorted by key, each value quoted. +pub(super) fn header_block(meta: &crate::ir::score::ScoreMetadata) -> Vec { + let named = [ + ("title", &meta.title), + ("subtitle", &meta.subtitle), + ("composer", &meta.composer), + ("arranger", &meta.arranger), + ("poet", &meta.lyricist), + ]; + let mut extra: Vec<(&String, &String)> = meta.extra.iter().collect(); + extra.sort(); + let fields: Vec<(&str, &str)> = named + .iter() + .filter_map(|(k, v)| Some((*k, v.as_deref()?))) + .chain(extra.iter().map(|(k, v)| (k.as_str(), v.as_str()))) + .collect(); + if fields.is_empty() { + return Vec::new(); + } + let mut lines = vec!["\\header {".to_string()]; + for (key, value) in fields { + // A key that is no LilyPond identifier (letters, with `-` or `_` + // between them) is written quoted, as LilyPond allows. + let plain = key + .split(['-', '_']) + .all(|w| !w.is_empty() && w.chars().all(|c| c.is_ascii_alphabetic())); + let key = if plain { + key.to_string() + } else { + format!("\"{}\"", escape_ly_string(key)) + }; + lines.push(format!(" {key} = \"{}\"", escape_ly_string(value))); + } + lines.push("}".to_string()); + lines.push(String::new()); + lines +} + /// Pack space-separated `tokens` into lines no longer than ~72 characters, each /// prefixed with `pad`, appending the lines to `lines`. Empty input is a no-op. pub(super) fn push_wrapped(tokens: &[String], pad: &str, lines: &mut Vec) { diff --git a/src/adapters/ir_to_ly/lyrics.rs b/src/adapters/ir_to_ly/lyrics.rs index 8c329f5..a43a70e 100644 --- a/src/adapters/ir_to_ly/lyrics.rs +++ b/src/adapters/ir_to_ly/lyrics.rs @@ -220,7 +220,7 @@ pub(super) fn emit_lyrics_variable(part: &Part, staff_filter: Option, lines: /// Escape special characters in lyric text for LilyPond: quoted when /// `\lyricmode` would read it as something else — a duration (`0/0/1`, /// `2nd`), a hyphen or extender, a brace, Scheme, or several words. -fn escape_lyric_text(text: &str) -> String { +pub(super) fn escape_lyric_text(text: &str) -> String { let special = text.is_empty() || text.starts_with(|c: char| c.is_ascii_digit()) || matches!(text, "--" | "__" | "_") diff --git a/src/adapters/ir_to_ly/maps.rs b/src/adapters/ir_to_ly/maps.rs index 1de91a9..abf1751 100644 --- a/src/adapters/ir_to_ly/maps.rs +++ b/src/adapters/ir_to_ly/maps.rs @@ -194,7 +194,8 @@ pub(super) fn partial_to_ly(dur: &Duration) -> String { pub(super) fn tremolo_suffix(note: &Note) -> String { if note.tremolo_marks > 0 && !note.two_note_tremolo { let base_denom = *note.duration.base.denom() as u32; - let n = base_denom * (1u32 << note.tremolo_marks); + let marks = note.tremolo_marks.min(crate::ir::note::MAX_TREMOLO_MARKS); + let n = u64::from(base_denom) << marks; format!(":{n}") } else { String::new() diff --git a/src/adapters/ir_to_ly/mod.rs b/src/adapters/ir_to_ly/mod.rs index c0b6b9a..b53a831 100644 --- a/src/adapters/ir_to_ly/mod.rs +++ b/src/adapters/ir_to_ly/mod.rs @@ -152,36 +152,7 @@ fn emit_preamble(score: &Score, version: &str, lang: PitchLanguage, lines: &mut lines.push(format!("\\language \"{}\"", lang.as_str())); lines.push(String::new()); - let meta = &score.metadata; - let has_header = meta.title.is_some() - || meta.composer.is_some() - || meta.arranger.is_some() - || meta.lyricist.is_some(); - - if has_header { - let esc = helpers::escape_ly_string; - lines.push("\\header {".to_string()); - if let Some(t) = &meta.title { - lines.push(format!(" title = \"{}\"", esc(t))); - } - if let Some(s) = &meta.subtitle { - lines.push(format!(" subtitle = \"{}\"", esc(s))); - } - if let Some(c) = &meta.composer { - lines.push(format!(" composer = \"{}\"", esc(c))); - } - if let Some(a) = &meta.arranger { - lines.push(format!(" arranger = \"{}\"", esc(a))); - } - if let Some(l) = &meta.lyricist { - lines.push(format!(" poet = \"{}\"", esc(l))); - } - for (key, val) in &meta.extra { - lines.push(format!(" {key} = \"{}\"", esc(val))); - } - lines.push("}".to_string()); - lines.push(String::new()); - } + lines.extend(helpers::header_block(&score.metadata)); // Paper block / page layout emit_paper(score, lines); diff --git a/src/adapters/ir_to_ly/music_emit.rs b/src/adapters/ir_to_ly/music_emit.rs index 7bbc0cb..1d4e01e 100644 --- a/src/adapters/ir_to_ly/music_emit.rs +++ b/src/adapters/ir_to_ly/music_emit.rs @@ -10,7 +10,6 @@ use crate::ir::articulation::{LyricSyllable, StartStop, SyllabicType}; use crate::ir::language::{PitchLanguage, PitchMode}; use crate::ir::music::{ContextType, Music, MusicDocument, RepeatType}; use crate::ir::pitch::Pitch; -use crate::ir::score::ScoreMetadata; use super::maps::{ articulation_to_ly, clef_to_ly, duration_to_ly, key_to_ly, ornament_to_ly, partial_to_ly, @@ -58,7 +57,7 @@ pub(super) fn emit_music_document( lines.push(String::new()); // Header - emit_header(&doc.metadata, &mut lines); + lines.extend(super::helpers::header_block(&doc.metadata)); // Music content. Always emit absolute pitches: this emitter does not wrap // output in `\relative { }`, so emitting relative octave marks would be @@ -125,32 +124,6 @@ fn with_pickup(music: &Music, d: &crate::ir::duration::Duration) -> Music { m } -/// Emit metadata as a `\header` block. -fn emit_header(meta: &ScoreMetadata, lines: &mut Vec) { - let esc = super::helpers::escape_ly_string; - let mut header_lines: Vec = Vec::new(); - - if let Some(ref title) = meta.title { - header_lines.push(format!(" title = \"{}\"", esc(title))); - } - if let Some(ref composer) = meta.composer { - header_lines.push(format!(" composer = \"{}\"", esc(composer))); - } - if let Some(ref subtitle) = meta.subtitle { - header_lines.push(format!(" subtitle = \"{}\"", esc(subtitle))); - } - if let Some(ref arranger) = meta.arranger { - header_lines.push(format!(" arranger = \"{}\"", esc(arranger))); - } - - if !header_lines.is_empty() { - lines.push("\\header {".to_string()); - lines.extend(header_lines); - lines.push("}".to_string()); - lines.push(String::new()); - } -} - /// Emit a Music node recursively. fn emit_music(music: &Music, ctx: &mut EmitCtx, lines: &mut Vec) { match music { @@ -359,7 +332,11 @@ fn group_graces(children: &[Music]) -> Vec { }; match out.last_mut() { Some(Music::Grace { content: group, .. }) => { - let mut all = notes(group); + // Moved, not cloned: a long run of graces stays linear. + let mut all = match std::mem::replace(&mut **group, Music::Sequential(Vec::new())) { + Music::Sequential(v) => v, + other => vec![other], + }; all.extend(notes(content)); **group = Music::Sequential(all); } @@ -437,7 +414,7 @@ fn emit_context( ) { let type_name = ctx_type.ly_name(); let name_part = match name { - Some(n) => format!(" = \"{n}\""), + Some(n) => format!(" = \"{}\"", super::helpers::escape_ly_string(n)), None => String::new(), }; @@ -524,7 +501,7 @@ fn emit_addlyrics(content: &Music, ctx: &EmitCtx, lines: &mut Vec) { for syllables in by_verse.values() { let mut tokens: Vec = Vec::new(); for syl in syllables { - let text = escape_lyric(&syl.text); + let text = super::lyrics::escape_lyric_text(&syl.text); match syl.syllabic { SyllabicType::Begin | SyllabicType::Middle => { tokens.push(text); @@ -544,15 +521,6 @@ fn emit_addlyrics(content: &Music, ctx: &EmitCtx, lines: &mut Vec) { } } -/// Quote lyric text if it contains spaces or special characters. -fn escape_lyric(text: &str) -> String { - if text.contains(' ') || text.contains('"') || text.contains('\\') { - format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) - } else { - text.to_string() - } -} - /// Emit a barline. fn emit_barline( barline: &crate::ir::direction::Barline, @@ -679,7 +647,8 @@ fn annotations_to_ly(annotations: &[Annotation]) -> String { Annotation::Glissando(StartStop::Start) => parts.push("\\glissando".to_string()), Annotation::Glissando(_) => {} Annotation::Tremolo { marks } => { - let denom = 1u32 << (marks + 2); // marks=1 → :8, marks=2 → :16, etc. + // marks=1 → :8, marks=2 → :16, etc. + let denom = 1u32 << ((*marks).min(crate::ir::note::MAX_TREMOLO_MARKS) + 2); parts.push(format!(":{denom}")); } Annotation::PedalStart => parts.push("\\sustainOn".to_string()), @@ -695,15 +664,7 @@ fn annotations_to_ly(annotations: &[Annotation]) -> String { Annotation::Fingering(f) => parts.push(format!("-{f}")), Annotation::Lyric(_) => {} // lyrics handled separately Annotation::Velocity(_) => {} // performance data, no notation - Annotation::OctaveShift(os) => { - let n = os.size; - if os.shift_type == "up" || os.shift_type == "down" { - let dir = if n > 0 { n } else { -n }; - parts.push(format!("\\ottava #{dir}")); - } else { - parts.push("\\ottava #0".to_string()); - } - } + Annotation::OctaveShift(os) => parts.push(format!("\\ottava #{}", os.octaves())), } } diff --git a/src/adapters/ir_to_ly/parts.rs b/src/adapters/ir_to_ly/parts.rs index 7178511..83b68ea 100644 --- a/src/adapters/ir_to_ly/parts.rs +++ b/src/adapters/ir_to_ly/parts.rs @@ -210,7 +210,7 @@ pub(super) fn emit_part_variable( // MIDI instrument setting (emitted at the top of the part variable body) let midi_set = if !part.midi_instrument.is_empty() { - let name = part.midi_instrument.to_lowercase(); + let name = super::helpers::escape_ly_string(&part.midi_instrument.to_lowercase()); format!(" \\set Staff.midiInstrument = \"{name}\"\n") } else { String::new() diff --git a/src/adapters/ir_to_mxml/direction.rs b/src/adapters/ir_to_mxml/direction.rs index c599ba8..7ab8f5d 100644 --- a/src/adapters/ir_to_mxml/direction.rs +++ b/src/adapters/ir_to_mxml/direction.rs @@ -167,7 +167,7 @@ impl IrToMxmlAdapter { content: mxml::DirectionTypeContents::OctaveShift(mxml::OctaveShift { attributes: mxml::OctaveShiftAttributes { r#type: shift_type, - size: Some(mdt::PositiveInteger(os.size as u32)), + size: (os.size > 0).then_some(mdt::PositiveInteger(os.size as u32)), ..Default::default() }, content: (), diff --git a/src/adapters/ir_to_mxml/helpers.rs b/src/adapters/ir_to_mxml/helpers.rs index d8b391b..c20ba04 100644 --- a/src/adapters/ir_to_mxml/helpers.rs +++ b/src/adapters/ir_to_mxml/helpers.rs @@ -57,10 +57,17 @@ pub(super) fn format_float(val: f64) -> String { // Division auto-computation // --------------------------------------------------------------------------- +/// Divisions used when no `u16` value represents every duration exactly (the +/// lcm of several coprime tuplets, 3·5·7·9·11·13 = 135135 already, does not +/// fit): 10080 = 2^5·3^2·5·7 is exact for every tuplet up to 10 and for many +/// beyond, and other durations are rounded. +const FALLBACK_DIVISIONS: u16 = 10080; + /// Compute divisions per quarter note that exactly represent all durations in -/// the score (including tuplets and short durations). +/// the score (including tuplets and short durations), or +/// [`FALLBACK_DIVISIONS`] when that number does not fit MusicXML's `u16`. pub(super) fn compute_score_divisions(score: &Score, base: u16) -> u16 { - let mut result = base as u64; + let mut result = Some(u64::from(base.max(1))); for part in score.parts() { for measure in &part.measures { for voice in &measure.voices { @@ -71,20 +78,22 @@ pub(super) fn compute_score_divisions(score: &Score, base: u16) -> u16 { VoiceElement::Chord(c) => &c.duration, }; if dur.tuplet_actual > 1 { - result = lcm_u64(result, dur.tuplet_actual as u64); + result = result.and_then(|r| lcm_u64(r, dur.tuplet_actual as u64)); } let base_n = *dur.base.numer(); let base_d = *dur.base.denom(); let g = gcd_u64(base_d.unsigned_abs(), (4 * base_n).unsigned_abs()); - let needed = base_d.unsigned_abs() / g; + let needed = base_d.unsigned_abs() / g.max(1); if needed > 1 { - result = lcm_u64(result, needed); + result = result.and_then(|r| lcm_u64(r, needed)); } } } } } - result as u16 + result + .and_then(|r| u16::try_from(r).ok()) + .unwrap_or(FALLBACK_DIVISIONS) } fn gcd_u64(mut a: u64, mut b: u64) -> u64 { @@ -96,6 +105,7 @@ fn gcd_u64(mut a: u64, mut b: u64) -> u64 { a } -fn lcm_u64(a: u64, b: u64) -> u64 { - a / gcd_u64(a, b) * b +/// `None` when the lcm overflows `u64`. +fn lcm_u64(a: u64, b: u64) -> Option { + (a / gcd_u64(a, b)).checked_mul(b) } diff --git a/src/adapters/ir_to_mxml/part.rs b/src/adapters/ir_to_mxml/part.rs index 7a28def..c65bea1 100644 --- a/src/adapters/ir_to_mxml/part.rs +++ b/src/adapters/ir_to_mxml/part.rs @@ -249,9 +249,10 @@ impl IrToMxmlAdapter { let mut fwd_pos: i64 = 0; for elem in &voice.elements { - // Interleave figured bass into voice 1's note stream. - if vi == 0 { - let cur_divs = fwd_pos as i32; + // Interleave figured bass into voice 1's note stream (a grace + // note takes no time: nothing new may be due). + let cur_divs = fwd_pos as i32; + if vi == 0 && cur_divs > fb_emitted_up_to { for (&off, fbs) in fb_by_offset.range(fb_emitted_up_to + 1..=cur_divs) { for fb in fbs { elements.push(mxml::MeasureElement::FiguredBass( diff --git a/src/adapters/ly_flatten.rs b/src/adapters/ly_flatten.rs index ab0c269..9066eff 100644 --- a/src/adapters/ly_flatten.rs +++ b/src/adapters/ly_flatten.rs @@ -7,7 +7,10 @@ //! Plain `\include` always inlines the referenced file (faithful to LilyPond's //! own behaviour). There is no include-once deduplication. If file A includes //! file B which in turn includes file A, a [`FlattenError::Circular`] is -//! returned. +//! returned. Without include-once, a diamond (a file including the next one +//! twice) doubles the output at every level, so flattening stops with +//! [`FlattenError::TooLarge`] after [`MAX_INCLUDES`] includes or +//! [`MAX_OUTPUT_BYTES`] of output. //! //! # Normalization (post-processing) //! After full expansion the output is scanned for duplicate command lines: @@ -31,6 +34,11 @@ use std::path::{Path, PathBuf}; use thiserror::Error; +/// Most `\include` expansions one flatten performs; real projects use tens. +pub const MAX_INCLUDES: usize = 10_000; +/// Most bytes of source one flatten emits. +pub const MAX_OUTPUT_BYTES: usize = 64 << 20; + // --------------------------------------------------------------------------- // Public API types // --------------------------------------------------------------------------- @@ -75,6 +83,9 @@ pub enum FlattenError { #[error("multiple \\header blocks found in flattened output")] MultipleHeaders, + + #[error("{0}; lytk refuses input this large")] + TooLarge(String), } // --------------------------------------------------------------------------- @@ -101,6 +112,8 @@ pub fn flatten_str(src: &str, base_dir: &Path, opts: FlattenOpts) -> Result { /// Canonical absolute paths of files currently open (ancestor chain). /// Used for circular-dependency detection. ancestors: Vec, + /// Includes expanded and bytes read so far, against the bounds above. + includes: usize, + bytes: usize, } fn expand(src: &str, base_dir: &Path, ctx: &mut ExpandCtx<'_>) -> Result { @@ -166,6 +182,19 @@ fn expand(src: &str, base_dir: &Path, ctx: &mut ExpandCtx<'_>) -> Result MAX_INCLUDES { + return Err(FlattenError::TooLarge(format!( + "the includes expand more than {MAX_INCLUDES} times" + ))); + } + if ctx.bytes > MAX_OUTPUT_BYTES { + return Err(FlattenError::TooLarge(format!( + "the flattened source exceeds {} MiB", + MAX_OUTPUT_BYTES >> 20 + ))); + } let inc_base = resolved.parent().unwrap_or(Path::new(".")); ctx.ancestors.push(canonical.clone()); @@ -532,6 +561,19 @@ mod tests { ); } + #[test] + fn test_include_diamond_is_refused() { + // f0 includes f1 twice, f1 includes f2 twice, …: 2^20 expansions. + let dir = TempDir::new().unwrap(); + for i in 0..20 { + let next = format!("\\include \"f{}.ly\"\n", i + 1); + write(dir.path(), &format!("f{i}.ly"), &next.repeat(2)); + } + write(dir.path(), "f20.ly", "c4\n"); + let err = flatten(&dir.path().join("f0.ly"), FlattenOpts::default()).unwrap_err(); + assert!(matches!(err, FlattenError::TooLarge(_)), "got {err:?}"); + } + #[test] fn test_self_include_error() { let dir = TempDir::new().unwrap(); diff --git a/src/adapters/ly_to_ir/builtins.rs b/src/adapters/ly_to_ir/builtins.rs new file mode 100644 index 0000000..6cc5b22 --- /dev/null +++ b/src/adapters/ly_to_ir/builtins.rs @@ -0,0 +1,1354 @@ +//! The commands LilyPond 2.26.0 defines, without their backslash: +//! lexer keywords, the identifiers of its `ly/` init files and its markup +//! commands (1347). Generated: `python3 scripts/ly_builtins.py lilypond v2.26.0`. + +/// Sorted, for binary search. +pub(super) const BUILTINS: &[&str] = &[ + "!", + "%", + "(", + ")", + "*", + "-", + "<", + "=", + ">", + "A", + "B", + "C", + "ChoirStaff", + "ChordGrid", + "ChordGridScore", + "ChordNames", + "CueVoice", + "Devnull", + "DrumStaff", + "DrumVoice", + "Dynamics", + "EnableGregorianDivisiones", + "FiguredBass", + "FretBoards", + "Gbarluadh", + "GbarluadhG", + "Gcrun", + "Gdarodo", + "Gfifteenthcutting", + "GfifteenthcuttingG", + "Global", + "GrandStaff", + "GregorianTranscriptionLyrics", + "GregorianTranscriptionStaff", + "GregorianTranscriptionVoice", + "Gseventeenthcutting", + "GseventeenthcuttingG", + "Gtaor", + "Gthrwd", + "IIJ", + "IJ", + "InternalGregorianStaff", + "InternalMensuralStaff", + "KievanStaff", + "KievanVoice", + "Lyrics", + "MensuralStaff", + "MensuralVoice", + "NoteNames", + "NullVoice", + "OneStaff", + "PetrucciStaff", + "PetrucciVoice", + "Piano", + "PianoStaff", + "RemoveAllEmptyStaves", + "RemoveEmptyStaves", + "RhythmicStaff", + "SATB", + "SSAATTBB", + "Score", + "Staff", + "StaffGroup", + "StandaloneRhythmScore", + "StandaloneRhythmStaff", + "StandaloneRhythmVoice", + "TabStaff", + "TabVoice", + "TwentyFourTETglyphs", + "VaticanaLyrics", + "VaticanaScore", + "VaticanaStaff", + "VaticanaVoice", + "Voice", + "[", + "]", + "^", + "abs-fontsize", + "abs-hspace", + "abs-vspace", + "absolute", + "absolute-volume-alist", + "accent", + "accentus", + "accepts", + "acciaccatura", + "accidental", + "accidentalStyle", + "acem", + "acemasiran", + "acembuselik", + "acemkurdi", + "acemliyegah", + "addChordShape", + "addInstrumentDefinition", + "addQuote", + "addToTagGroup", + "addlyrics", + "aeolian", + "after", + "afterGrace", + "afterGraceFraction", + "aikenHeads", + "aikenHeadsMinor", + "aikenThinHeads", + "aikenThinHeadsMinor", + "alias", + "align-on-other", + "allowBreak", + "allowPageTurn", + "allowVoltaHook", + "alterBroken", + "alternative", + "ambitusAfter", + "anberefsan", + "annotate-moving", + "append-to-tag", + "appendToTag", + "appendToTagMarkup", + "applyContext", + "applyMusic", + "applyOutput", + "applySwing", + "applySwingWithOffset", + "appoggiatura", + "approximatePitch", + "arabanbuselik", + "arabicStringNumbers", + "aramican", + "arazbar", + "arazbarbuselik", + "arazbarek", + "arazbarzemzeme", + "arpeggio", + "arpeggioArrowDown", + "arpeggioArrowUp", + "arpeggioBracket", + "arpeggioNormal", + "arpeggioParenthesis", + "arpeggioParenthesisDashed", + "arrow-head", + "articulate", + "ascendens", + "askevza", + "assertBeamQuant", + "assertBeamSlope", + "atLeft", + "atRight", + "auctum", + "augmentum", + "auto-footnote", + "autoBeamOff", + "autoBeamOn", + "autoBreaksOff", + "autoBreaksOn", + "autoChange", + "autoLineBreaksOff", + "autoLineBreaksOn", + "autoPageBreaksOff", + "autoPageBreaksOn", + "bachschleifer", + "backslashed-digit", + "bahrinazik", + "balloonGrobText", + "balloonLengthOff", + "balloonLengthOn", + "balloonText", + "bar", + "bar-line", + "barNumberCheck", + "bari", + "barluadh", + "barluadhG", + "bassFigureExtendersOff", + "bassFigureExtendersOn", + "bassFigureStaffAlignmentDown", + "bassFigureStaffAlignmentNeutral", + "bassFigureStaffAlignmentUp", + "bayati", + "beam", + "beamExceptions", + "bendAfter", + "bendHold", + "bendStartLevel", + "bendihisar", + "bestehicaz", + "bestehisar", + "besteisfahan", + "bestenigar", + "beyati", + "beyatiaraban", + "beyatiarabanbuselik", + "beyatibuselik", + "beyatiruyiacem", + "bezmitarab", + "bgrip", + "bigger", + "binding-offset-default", + "birl", + "blackTriangleMarkup", + "blank-after-score-page-penalty", + "blank-last-page-penalty", + "blank-page-penalty", + "bold", + "book", + "bookOutputName", + "bookOutputSuffix", + "bookTitleMarkup", + "bookpart", + "bottom-margin-default", + "box", + "bp", + "bracket", + "break", + "breakAlignInsert", + "breakDynamicSpan", + "breathe", + "breve", + "btaor", + "buselik", + "buselikasiran", + "buzurk", + "cad", + "cadenzaOff", + "cadenzaOn", + "caesura", + "canfeza", + "caps", + "cargah", + "cargaheski", + "catcha", + "catchb", + "catchc", + "catchd", + "catche", + "cavum", + "center", + "center-align", + "center-column", + "chahargah", + "change", + "char", + "check-consistency", + "chordRepeats", + "chordSlur", + "chordmode", + "chordmodifiers", + "chords", + "circle", + "circulus", + "clef", + "cm", + "coda", + "codaMark", + "column", + "column-lines", + "combine", + "command-name", + "compound-meter", + "compressEmptyMeasures", + "compressMMRests", + "concat", + "conditional-trill-markup", + "consists", + "context", + "contextPropertyCheck", + "cr", + "cresc", + "crescHairpin", + "crescTextCresc", + "crossStaff", + "crun", + "crunamb", + "crunambfosg", + "crunamc", + "crunamcfosg", + "crunamd", + "crunamdfosg", + "cueClef", + "cueClefUnset", + "cueDuring", + "cueDuringWithClef", + "customTabClef", + "dare", + "dari", + "darodo", + "dashBang", + "dashDash", + "dashDot", + "dashHat", + "dashLarger", + "dashPlus", + "dashUnderscore", + "dbirl", + "dblA", + "dblG", + "dbla", + "dblb", + "dblc", + "dbld", + "dble", + "dblf", + "dblg", + "dcrun", + "deadNote", + "deadNotesOff", + "deadNotesOn", + "decr", + "decresc", + "default", + "defaultNoteHeads", + "defaultStringTunings", + "defaultTimeSignature", + "defaultchild", + "defineBarLine", + "delkashMahur", + "deminutum", + "denies", + "descendens", + "description", + "dilefruz", + "dilfuruz", + "dilkes", + "dilkeshaveran", + "dilkeside", + "dilkusa", + "dilnevaz", + "dilnisin", + "dim", + "dimHairpin", + "dimTextDecr", + "dimTextDecresc", + "dimTextDim", + "dir-column", + "discant", + "displayLilyMusic", + "displayMusic", + "displayScheme", + "divisioMaior", + "divisioMaxima", + "divisioMinima", + "dorian", + "dotsDown", + "dotsNeutral", + "dotsUp", + "doubleflat", + "doublesharp", + "down", + "downbow", + "downmordent", + "downprall", + "draw-circle", + "draw-dashed-line", + "draw-dotted-line", + "draw-hline", + "draw-line", + "draw-squiggle-line", + "dre", + "dropNote", + "drumPitchNames", + "drummode", + "drums", + "dugah", + "dugahbuselik", + "dugahmaye", + "dwn", + "dynamic", + "dynamicDown", + "dynamicNeutral", + "dynamicUp", + "easyHeadsOff", + "easyHeadsOn", + "egrip", + "ellipse", + "enablePerStaffTiming", + "endSkipNCs", + "endSpanners", + "endcr", + "endcresc", + "enddecr", + "enddecresc", + "enddim", + "englishChords", + "episemFinis", + "episemInitium", + "epsfile", + "esfahan", + "eskisipihr", + "espressivo", + "etc", + "evcara", + "evcibahrinazik", + "evciruyineva", + "evcisevk", + "evenHeaderMarkup", + "eventChords", + "evic", + "evicbuselik", + "evichuzi", + "evicmaye", + "expandEmptyMeasures", + "eyeglasses", + "f", + "featherDurations", + "ferahfeza", + "ferahnak", + "ferahnakasiran", + "ferahnuma", + "fermata", + "fethibelgrad", + "fethidil", + "feyziyekta", + "ff", + "fff", + "ffff", + "fffff", + "fifteenthcutting", + "fifteenthcuttingG", + "figured-bass", + "figuremode", + "figures", + "fill-line", + "fill-with-pattern", + "filled-box", + "finalis", + "fine", + "finger", + "first-page-number", + "first-visible", + "fixed", + "flagStyleDefault", + "flagStyleStacked", + "flageolet", + "flat", + "flexa", + "fontCaps", + "fontsize", + "footnote", + "footnote-footer-padding", + "footnote-number-raise", + "footnote-numbering-function", + "footnote-padding", + "footnote-separator-markup", + "four-string-banjo", + "fp", + "fraction", + "freeBass", + "frenchChords", + "fret-diagram", + "fret-diagram-terse", + "fret-diagram-verbose", + "fromproperty", + "funkHeads", + "funkHeadsMinor", + "fz", + "gbirl", + "general-align", + "gerdaniye", + "gerdaniyebuselik", + "gerdaniyekurdi", + "germanChords", + "gestuguzaribahar", + "gevest", + "glide", + "glissando", + "grA", + "grG", + "grGcad", + "gra", + "gracad", + "grace", + "graphvizReleaseCallbacks", + "graphvizSetupCallbacks", + "graphvizWriteGraph", + "grb", + "grc", + "grd", + "gre", + "grf", + "grg", + "grip", + "gripthrwd", + "grobdescriptions", + "gulbuse", + "guldeste", + "gulizar", + "gulsenivefa", + "halfBeaming", + "halfopen", + "halign", + "harmonic", + "harmonicByFret", + "harmonicByRatio", + "harmonicNote", + "harmonicsOff", + "harmonicsOn", + "harp-pedal", + "haydnturn", + "hbracket", + "hcad", + "hcatcha", + "hcatchb", + "hcatchc", + "hcatchd", + "hcatche", + "hcenter-in", + "hdblA", + "hdblG", + "hdbla", + "hdblb", + "hdblc", + "hdbld", + "hdble", + "hdblf", + "hdblg", + "header", + "heel", + "heelcircle", + "helArabicPitchNames", + "henzelongfermata", + "henzeshortfermata", + "hicaz", + "hicazasiran", + "hicazbuselik", + "hicazeyn", + "hicazhumayun", + "hicazirak", + "hicazkar", + "hicazuzzal", + "hicazzemzeme", + "hicazzirgule", + "hide", + "hideKeySignature", + "hideNotes", + "hideSplitTiedTabNotes", + "hideStaffSwitch", + "hijaz", + "hijaz_kar", + "hisar", + "hisarbuselik", + "hisarvechisehnaz", + "hizaz", + "horasan", + "hosaver", + "houssaini", + "hshakeA", + "hshakea", + "hshakeb", + "hshakec", + "hshaked", + "hshakee", + "hshakef", + "hshakeg", + "hslurA", + "hslura", + "hslurb", + "hslurc", + "hslurd", + "hslure", + "hslurf", + "hslurg", + "hspace", + "huge", + "huseyni", + "huseyniaraban", + "huseyniasiran", + "huseynizemzeme", + "huzam", + "huzi", + "huzzam", + "ictus", + "if", + "ignatzekExceptionMusic", + "ignatzekExceptions", + "iij", + "ij", + "image", + "improvisationOff", + "improvisationOn", + "in", + "in-note-padding", + "in-note-system-padding", + "inStaffSegno", + "incipit", + "inclinatum", + "include", + "indent-default", + "inherit-acceptability", + "initialContextFrom", + "inner-margin-default", + "instrument-definitions", + "instrument-equalizer-alist", + "instrumentSwitch", + "inversion", + "invertChords", + "ionian", + "irak", + "irakasiran", + "iraq", + "isfahan", + "isfahanek", + "isfahanruyineva", + "italianChords", + "italic", + "jaharkah", + "jump", + "justified-lines", + "justify", + "justify-field", + "justify-line", + "justify-string", + "karcigar", + "karjkhar", + "keep-with-tag", + "keepWithTag", + "key", + "keyAlterationOrder", + "kievanOff", + "kievanOn", + "killCues", + "kucek", + "kurd", + "kurdi", + "kurdilihicazkar", + "label", + "laissezVibrer", + "lalegul", + "language", + "languageRestore", + "languageSaveAndChange", + "large", + "larger", + "last-bottom-spacing", + "layout", + "left", + "left-align", + "left-brace", + "left-column", + "left-margin-default", + "lheel", + "lheeltoe", + "ligature", + "line", + "linea", + "lineprall", + "locrian", + "longa", + "longfermata", + "lookup", + "lower", + "ltoe", + "ltoeheel", + "lydian", + "lyricmode", + "lyrics", + "lyricsto", + "magnify", + "magnifyMusic", + "magnifyStaff", + "mahur", + "mahurbuselik", + "mahurhan", + "maininput", + "major", + "makamGlyphs", + "makamPitchNames", + "make-one-stanza", + "make-one-voice-staff", + "make-one-voice-vocal-staff", + "make-pianostaff", + "make-two-vocal-staves-with-stanzas", + "make-two-voice-staff", + "make-two-voice-vocal-staff", + "make-voice", + "makeClusters", + "makeDefaultStringTuning", + "map-markup-commands", + "marcato", + "marchTime", + "mark", + "markLengthOff", + "markLengthOn", + "markalphabet", + "markletter", + "markup", + "markup-markup-spacing", + "markup-system-spacing", + "markupMap", + "markuplist", + "maveraunnehir", + "maxima", + "mayeasiran", + "medianChordGridStyle", + "melisma", + "melismaEnd", + "mergeDifferentlyDottedOff", + "mergeDifferentlyDottedOn", + "mergeDifferentlyHeadedOff", + "mergeDifferentlyHeadedOn", + "mf", + "midi", + "midiDrumPitches", + "minor", + "mixolydian", + "mm", + "modalInversion", + "modalTranspose", + "mokhalefsegah", + "mordent", + "mp", + "muberka", + "muhalif", + "muhalifiirak", + "muhayyer", + "muhayyerbuselik", + "muhayyerkurdi", + "muhayyersunbule", + "muhayyerzirgule", + "multi-measure-rest-by-number", + "musicLength", + "musicMap", + "musicglyph", + "mustear", + "mycommand", + "n", + "nahawande", + "nakriz", + "name", + "narefte", + "natural", + "nawaatar", + "neva", + "nevabagdat", + "nevabuselik", + "nevadilkes", + "nevakurdi", + "neveda", + "neveser", + "nevkes", + "nevruz", + "nevruzirumi", + "new", + "newSpacingSection", + "nigar", + "nihavend", + "nihavendikebir", + "nikriz", + "nisabur", + "nisaburek", + "nisaburrumi", + "noBeam", + "noBreak", + "noPageBreak", + "noPageTurn", + "nonArpeggiato", + "normal-size-sub", + "normal-size-super", + "normal-text", + "normal-weight", + "normalsize", + "note", + "note-by-number", + "notemode", + "nuhuft", + "null", + "number", + "numericTimeSignature", + "octaveCheck", + "oddFooterMarkup", + "oddHeaderMarkup", + "offset", + "omit", + "on-the-fly", + "once", + "oneVoice", + "open", + "oriscus", + "ottava", + "outer-margin-default", + "output-scale", + "oval", + "overlay", + "override", + "override-lines", + "overrideProperty", + "overrideTimeSignatureSettings", + "overtie", + "p", + "pGdarodo", + "pad-around", + "pad-markup", + "pad-to-box", + "pad-x", + "pad-x-left", + "pad-x-right", + "page-breaking", + "page-link", + "page-number-type", + "page-ref", + "pageBreak", + "pageTurn", + "palmMute", + "palmMuteOff", + "palmMuteOn", + "paper", + "parallelMusic", + "parenthesize", + "partCombine", + "partCombineApart", + "partCombineAutomatic", + "partCombineChords", + "partCombineDown", + "partCombineForce", + "partCombineListener", + "partCombineSoloI", + "partCombineSoloII", + "partCombineUnisono", + "partCombineUp", + "partial", + "path", + "pattern", + "pdarodo", + "pencgah", + "persianAltKeyAlterationOrder", + "persianGlyphs", + "persianKeyAlterationOrder", + "persianKeySignaturePaddingPairs", + "persianPitchNames", + "pes", + "pesendide", + "peykinesat", + "peykisafa", + "pgrace", + "phrasingSlurDashPattern", + "phrasingSlurDashed", + "phrasingSlurDotted", + "phrasingSlurDown", + "phrasingSlurHalfDashed", + "phrasingSlurHalfSolid", + "phrasingSlurNeutral", + "phrasingSlurSolid", + "phrasingSlurUp", + "phrygian", + "pitchedTrill", + "pitchnamesBagpipe", + "pointAndClickOff", + "pointAndClickOn", + "pointAndClickTypes", + "polygon", + "polymetric", + "popContextProperty", + "portato", + "postscript", + "pp", + "ppp", + "pppp", + "ppppp", + "prall", + "pralldown", + "prallmordent", + "prallprall", + "prallup", + "preBend", + "preBendHold", + "predefinedFretboardsOff", + "predefinedFretboardsOn", + "previous-pitchnames", + "print-first-page-number", + "print-page-number", + "property-recursive", + "propertyOverride", + "propertyRevert", + "propertySet", + "propertyTweak", + "propertyUnset", + "pt", + "pthrwd", + "push-to-tag", + "pushContextProperty", + "pushToTag", + "pushToTagMarkup", + "put-adjacent", + "qr-code", + "quarterBeaming", + "quilisma", + "quoteDuring", + "ragged-bottom", + "ragged-last-bottom", + "rahatulervah", + "raise", + "raiseNote", + "rast", + "rastasiran", + "rasthaveran", + "rasticedid", + "rastisagir", + "rastmaye", + "reduceChords", + "reelTime", + "rehavi", + "rekbizavil", + "relative", + "remove", + "remove-with-tag", + "removeFromTagGroup", + "removeWithTag", + "rengidil", + "repeat", + "repeatTie", + "replace", + "replace-with-tag", + "replaceWithTag", + "replaceWithTagMarkup", + "reset-footnotes-on-new-page", + "resetRelativeOctave", + "resetTagGroup", + "resetTagGroups", + "responsum", + "rest", + "rest-by-number", + "retrograde", + "reverseturn", + "revert", + "revertTimeSignatureSettings", + "revnaknuma", + "rfz", + "rheel", + "rheeltoe", + "rhythm", + "right", + "right-align", + "right-brace", + "right-column", + "right-margin-default", + "rightHandFinger", + "romanStringNumbers", + "rotate", + "rounded-box", + "rtoe", + "rtoeheel", + "ruhnuvaz", + "ruyidilara", + "ruyiirak", + "saba", + "sabaasiran", + "sababuselik", + "sabazemzeme", + "sacredHarpHeads", + "sacredHarpHeadsMinor", + "sans", + "sazkar", + "scale", + "scaleDurations", + "score", + "score-lines", + "score-markup-spacing", + "score-system-spacing", + "scoreTitleMarkup", + "section", + "sectionLabel", + "sedaraban", + "segah", + "segaharaban", + "segahmaye", + "segno", + "segnoMark", + "sehnaz", + "sehnazasiran", + "sehnazbuselik", + "sehnazhaveran", + "selmek", + "semiGermanChords", + "semicirculus", + "semiflat", + "semisharp", + "senzaMisuraTimeSignatureOff", + "senzaMisuraTimeSignatureX", + "sequential", + "serefnuma", + "serif", + "sesquiflat", + "sesquisharp", + "set", + "set-symbol", + "setDefaultDurationToQuarter", + "seventeenthcutting", + "seventeenthcuttingG", + "sevkaver", + "sevkefza", + "sevkicihan", + "sevkidil", + "sevkiserab", + "sevkitarab", + "sevkutarab", + "sf", + "sff", + "sfp", + "sfz", + "shakeA", + "shakea", + "shakeb", + "shakec", + "shaked", + "shakee", + "shakef", + "shakeg", + "shape", + "sharp", + "shiftDurations", + "shiftOff", + "shiftOn", + "shiftOnn", + "shiftOnnn", + "short-indent-default", + "shortfermata", + "showKeySignature", + "showSplitTiedTabNotes", + "showStaffSwitch", + "shur", + "shurGvlat", + "shurk", + "signumcongruentiae", + "sikah", + "simple", + "simultaneous", + "single", + "sipihr", + "sivenuma", + "skip", + "skipNC", + "skipNCs", + "slashSeparator", + "slashed-digit", + "slashedGrace", + "slashturn", + "slurA", + "slurDashPattern", + "slurDashed", + "slurDotted", + "slurDown", + "slurHalfDashed", + "slurHalfSolid", + "slurNeutral", + "slurSolid", + "slurUp", + "slura", + "slurb", + "slurc", + "slurd", + "slure", + "slurf", + "slurg", + "small", + "smallCaps", + "smaller", + "snappizzicato", + "sostenutoOff", + "sostenutoOn", + "sourcefileline", + "sourcefilename", + "southernHarmonyHeads", + "southernHarmonyHeadsMinor", + "souznak", + "sp", + "spp", + "staccatissimo", + "staccato", + "staffHighlight", + "start", + "startAcciaccaturaMusic", + "startAppoggiaturaMusic", + "startGraceMusic", + "startGraceSlur", + "startGroup", + "startMeasureCount", + "startMeasureSpanner", + "startOptionalMaterial", + "startSlashedGraceMusic", + "startStaff", + "startTextSpan", + "startTrillSpan", + "stdBass", + "stdBassIV", + "stdBassV", + "stdBassVI", + "stemDown", + "stemNeutral", + "stemUp", + "stemspace", + "stencil", + "stop", + "stopAcciaccaturaMusic", + "stopAppoggiaturaMusic", + "stopGraceMusic", + "stopGraceSlur", + "stopGroup", + "stopMeasureCount", + "stopMeasureSpanner", + "stopOptionalMaterial", + "stopSlashedGraceMusic", + "stopStaff", + "stopStaffHighlight", + "stopTextSpan", + "stopTrillSpan", + "stopped", + "storePredefinedDiagram", + "string-lines", + "stringTuning", + "stropha", + "strut", + "styledNoteHeads", + "sub", + "submeasureBarsOff", + "submeasureBarsOn", + "sultanicedid", + "sultaniirak", + "sultanirast", + "sultanisegah", + "sultaniyegah", + "super", + "sustainOff", + "sustainOn", + "suzidil", + "suzidilara", + "suznak", + "system-system-spacing", + "tabChordRepeats", + "tabChordRepetition", + "tabFullNotation", + "table", + "tag", + "tag-list", + "tagGroup", + "tagGroupRef", + "tagline", + "tahir", + "tahirbuselik", + "taor", + "taoramb", + "taoramc", + "taoramd", + "taorjmd", + "taorold", + "tarabengiz", + "tarzicedid", + "tarzicihan", + "tarzinevin", + "tavrinihavend", + "tcad", + "tcatcha", + "tcatchb", + "tcatchc", + "tcatchd", + "tcatche", + "tdblG", + "tdbla", + "tdblb", + "tdblc", + "tdbld", + "tdble", + "tdblf", + "tdblg", + "tebriz", + "teeny", + "tempo", + "temporary", + "tenuto", + "text-accidental", + "text-doubleflat", + "text-doublesharp", + "text-flat", + "text-natural", + "text-sharp", + "textEndMark", + "textLengthOff", + "textLengthOn", + "textMark", + "textSpannerDown", + "textSpannerNeutral", + "textSpannerUp", + "thcad", + "thrwd", + "thrwe", + "thrwf", + "thumb", + "tie", + "tieDashPattern", + "tieDashed", + "tieDotted", + "tieDown", + "tieHalfDashed", + "tieHalfSolid", + "tieNeutral", + "tieSolid", + "tieUp", + "tied-lyric", + "time", + "timeAbbrev", + "times", + "tiny", + "tocItem", + "tocItemWithDotsMarkup", + "toe", + "top-margin-default", + "top-markup-spacing", + "top-system-spacing", + "translate", + "translate-scaled", + "transparent", + "transpose", + "transposedCueDuring", + "transposition", + "treCorde", + "trebling", + "triangle", + "trill", + "tripleA", + "tripletFeel", + "tshakeA", + "tshakea", + "tshakeb", + "tshakec", + "tshaked", + "tshakee", + "tshakef", + "tshakeg", + "tslurA", + "tslura", + "tslurb", + "tslurc", + "tslurd", + "tslure", + "tslurf", + "tslurg", + "tuplet", + "tupletDown", + "tupletNeutral", + "tupletSpan", + "tupletUp", + "turkishMakamGlyphs", + "turkishMakamPitchNames", + "turn", + "tweak", + "two-sided", + "txcrun", + "txcrunam", + "txtaor", + "txtaoram", + "txtaorcrun", + "txtaorcrunam", + "type", + "typewriter", + "unHideNotes", + "unaCorda", + "underline", + "undertie", + "undo", + "unfoldRepeats", + "unfolded", + "unless", + "unset", + "up", + "upbow", + "upmordent", + "upprall", + "upright", + "ussak", + "ussakasiran", + "uzzal", + "varcoda", + "varheel", + "vartoe", + "vcenter", + "vecdidil", + "vechiarazbar", + "verbatim-file", + "version", + "versus", + "verylongfermata", + "veryshortfermata", + "virga", + "virgula", + "voiceFour", + "voiceFourStyle", + "voiceNeutralStyle", + "voiceOne", + "voiceOneStyle", + "voiceThree", + "voiceThreeStyle", + "voiceTwo", + "voiceTwoStyle", + "voices", + "void", + "volta", + "volta-number", + "vowelTransition", + "vshape", + "vspace", + "walkerHeads", + "walkerHeadsMinor", + "wbirl", + "wcatchd", + "whcatchd", + "whiteCircleMarkup", + "whiteTriangleMarkup", + "whiteout", + "whshaked", + "whslurd", + "with", + "with-color", + "with-dimension", + "with-dimension-from", + "with-dimensions", + "with-dimensions-from", + "with-link", + "with-outline", + "with-string-transformer", + "with-true-dimension", + "with-true-dimensions", + "with-url", + "withMusicProperty", + "withRelativeDir", + "woodwind-diagram", + "wordwrap", + "wordwrap-field", + "wordwrap-internal", + "wordwrap-lines", + "wordwrap-string", + "wordwrap-string-internal", + "wshaked", + "wslurd", + "wtcatchd", + "wthrwe", + "wtshaked", + "wtslurd", + "xNote", + "xNotesOff", + "xNotesOn", + "yegah", + "yenisipihr", + "zavil", + "zavilasiran", + "zevkidil", + "zirefkend", + "zirgulelisuznak", + "|", + "~", +]; diff --git a/src/adapters/ly_to_ir/consume.rs b/src/adapters/ly_to_ir/consume.rs index ecd3c7d..456dca0 100644 --- a/src/adapters/ly_to_ir/consume.rs +++ b/src/adapters/ly_to_ir/consume.rs @@ -73,20 +73,36 @@ pub(super) fn consume_duration( ) -> Duration { // Look for unsigned_integer if *i < children.len() && children[*i].kind() == "unsigned_integer" { - let num_text = state.text(children[*i]).to_string(); + let node = children[*i]; + let num_text = state.text(node); *i += 1; - if let Ok(num) = num_text.parse::() { - let dots = consume_dots(state, children, i); - // Convert LilyPond number to fraction: 4 → 1/4, 2 → 1/2, 1 → 1/1 - let base = if num > 0 { - Ratio::new(1i64, num as i64) - } else { + let dots = consume_dots(state, children, i); + // Convert LilyPond number to fraction: 4 → 1/4, 2 → 1/2, 1 → 1/1. + // Only powers of two are durations (LilyPond reports anything else + // as "not a duration" and reads a quarter, as here); past 1024, the + // shortest value LilyPond draws, a denominator only feeds overflow. + let base = match num_text.parse::() { + Ok(num) if num.is_power_of_two() && num <= 1024 => Ratio::new(1i64, num as i64), + Ok(num) if num.is_power_of_two() => { + state.warn( + node, + "unsupported-value", + format!("lytk reads durations up to 1024: `{num}` is read as a quarter"), + ); Ratio::new(1, 4) - }; - let dur = Duration::dotted(base, dots); - state.last_duration = dur.clone(); - return dur; - } + } + _ => { + state.error( + node, + "invalid-duration", + format!("`{num_text}` is not a duration: read as a quarter"), + ); + Ratio::new(1, 4) + } + }; + let dur = Duration::dotted(base, dots); + state.last_duration = dur.clone(); + return dur; } // Long durations are escaped words: \breve, \longa, \maxima if *i < children.len() && children[*i].kind() == "escaped_word" { @@ -124,6 +140,11 @@ fn consume_dots(state: &WalkState, children: &[Node], i: &mut usize) -> u8 { dots } +/// Largest duration multiplier numerator (`R1*16777216`). Past it no note +/// value stays under the reader's length bound, and the bound keeps duration +/// arithmetic far from `i64` overflow. +pub(super) const MAX_MULTIPLIER: u32 = 1 << 24; + /// Consume an optional `*N` or `*N/M` duration scaling factor. /// Returns the fractional scale (e.g. `*8/7` → 8/7, `*3` → 3/1). /// Returns `None` if no multiplier is present. @@ -133,31 +154,60 @@ fn consume_dots(state: &WalkState, children: &[Node], i: &mut usize) -> u8 { /// - `punctuation("*")` `fraction("N/M")` (single fraction token), or /// - `punctuation("*")` `unsigned_integer("N")` `punctuation("/")` `unsigned_integer("M")` pub(super) fn consume_duration_scale( - state: &WalkState, + state: &mut WalkState, children: &[Node], i: &mut usize, ) -> Option { + // A multiplier numerator past MAX_MULTIPLIER stops the reading; a 0, + // unreadable or oversized denominator drops the multiplier. + let numer_or_refuse = |state: &mut WalkState, text: &str| -> Option { + match text.parse::() { + Ok(n) if n <= MAX_MULTIPLIER => Some(i64::from(n)), + _ => { + state.refuse(format!( + "duration multiplier *{text} is above the limit of {MAX_MULTIPLIER}" + )); + None + } + } + }; + let denom_of = |state: &mut WalkState, node: Node, text: &str| match text.parse::() { + Ok(d) if d != 0 && d <= MAX_MULTIPLIER => Some(i64::from(d)), + Ok(0) => { + state.error( + node, + "invalid-ratio", + "a duration multiplier with a zero denominator: dropped".to_string(), + ); + None + } + _ => { + state.warn( + node, + "unsupported-value", + format!("a duration multiplier denominator above {MAX_MULTIPLIER}: dropped"), + ); + None + } + }; if *i < children.len() && children[*i].kind() == "punctuation" { let ptext = punct_text(state, children[*i]); if ptext == "*" { *i += 1; // Case 1: fraction token (e.g. "8/7") if *i < children.len() && children[*i].kind() == "fraction" { - let frac_text = state.text(children[*i]); + let node = children[*i]; + let frac_text = state.text(node); *i += 1; - if let Some((num, den)) = parse_fraction(frac_text) { - if den == 0 { - return None; // *N/0 would panic in Frac::new - } - return Some(Frac::new(num as i64, den as i64)); - } - return None; + let (num, den) = frac_text.split_once('/')?; + let numer = numer_or_refuse(state, num)?; + return Some(Frac::new(numer, denom_of(state, node, den)?)); } // Case 2: unsigned_integer, optionally followed by / and unsigned_integer if *i < children.len() && children[*i].kind() == "unsigned_integer" { let num_text = state.text(children[*i]).to_string(); *i += 1; - let numer: i64 = num_text.parse().unwrap_or(1); + let numer = numer_or_refuse(state, &num_text)?; // Check for fraction: *N/M as separate tokens if *i + 1 < children.len() && children[*i].kind() == "punctuation" @@ -165,13 +215,10 @@ pub(super) fn consume_duration_scale( { *i += 1; // skip "/" if *i < children.len() && children[*i].kind() == "unsigned_integer" { - let denom_text = state.text(children[*i]).to_string(); + let node = children[*i]; + let denom_text = state.text(node); *i += 1; - let denom: i64 = denom_text.parse().unwrap_or(1); - if denom == 0 { - return None; // *N/0 would panic in Frac::new - } - return Some(Frac::new(numer, denom)); + return Some(Frac::new(numer, denom_of(state, node, denom_text)?)); } } return Some(Frac::from_integer(numer)); @@ -200,7 +247,8 @@ pub(super) fn consume_tremolo( if let Ok(n) = num_text.parse::() { // base_dur denominator: e.g. quarter = 1/4 → denom 4 let base_denom = *base_dur.base.denom() as u32; - if n > base_denom && base_denom > 0 { + // `:N` is a duration, as LilyPond reads it: up to 1024. + if n > base_denom && base_denom > 0 && n.is_power_of_two() && n <= 1024 { let ratio = n / base_denom; return (ratio as f64).log2() as u8; } @@ -343,6 +391,21 @@ pub(super) fn is_post_note_command(text: &str) -> bool { ) || is_dynamic_name(text) } +/// The sibling before `children[i]`, skipping comments. +pub(super) fn previous<'a>(children: &[Node<'a>], i: usize) -> Option> { + children[..i] + .iter() + .rev() + .find(|n| n.kind() != "comment") + .copied() +} + +/// The index past a markup expression starting at `i` (just after `\markup`); +/// see [`super::text::markup_end`]. +pub(super) fn skip_markup(state: &WalkState, children: &[Node], i: usize) -> usize { + super::text::markup_end(state.source, children, i) +} + /// Get the text of a punctuation node (which may have a child). pub(super) fn punct_text(state: &WalkState, node: Node) -> String { if node.child_count() > 0 { @@ -439,32 +502,15 @@ pub(super) fn ly_number_to_beat_unit(num: &str) -> String { .to_string() } -/// Extract the text content of a `string` node. Returns the unquoted value. +/// The decoded text of a `string` node (see [`super::text::string_value`]). pub(super) fn extract_string_value(state: &WalkState, node: Node) -> String { - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - if child.kind() == "string_fragment" { - return state.text(child).to_string(); - } - } - String::new() + super::text::string_value(state.source, node) } -/// Extract text content from a markup expression_block like `{ "pizz." }`. -/// Walks children looking for string nodes and returns the concatenated text. +/// The plain text of a markup block like `{ \italic dolce }`: its words and +/// strings (see [`super::text::markup_text`]). pub(super) fn extract_markup_text(state: &WalkState, block: Node) -> String { - let mut result = String::new(); - let mut cursor = block.walk(); - for child in block.children(&mut cursor) { - if child.kind() == "string" { - let s = extract_string_value(state, child); - if !result.is_empty() { - result.push(' '); - } - result.push_str(&s); - } - } - result + super::text::markup_text(state.source, &[block]) } /// Parse a `\with { ... }` expression block and extract known properties. @@ -623,6 +669,9 @@ pub(super) fn consume_override(state: &mut WalkState, children: &[Node], mut i: // Whatever follows the "=" is the value — grab its raw text value = state.text(node).to_string(); i += 1; + if matches!(value.as_str(), "\\markup" | "\\markuplist") { + i = skip_markup(state, children, i); + } break; } _ => break, @@ -777,7 +826,8 @@ pub(super) fn parse_paper_block(state: &mut WalkState, block: Node) { state.page_layout = Some(layout); } -/// Parse a fraction string like "4/4" into (numerator, denominator). +/// Parse a fraction string like "4/4" into (numerator, denominator). A zero +/// denominator is `None`: every caller would build a ratio from it. pub(super) fn parse_fraction(text: &str) -> Option<(u32, u32)> { let parts: Vec<&str> = text.split('/').collect(); if parts.len() == 2 { @@ -790,7 +840,7 @@ pub(super) fn parse_fraction(text: &str) -> Option<(u32, u32)> { } /// Parse `#(ly:make-moment N D)` from a Scheme expression text. -/// Returns (numerator, denominator) if successful. +/// Returns (numerator, denominator) if successful, a zero denominator included. pub(super) fn parse_ly_make_moment(text: &str) -> Option<(u32, u32)> { // Text looks like: #(ly:make-moment 3 4) or #(ly:make-moment 3/4) let inner = text.trim_start_matches('#').trim(); @@ -814,15 +864,10 @@ pub(super) fn parse_ly_make_moment(text: &str) -> Option<(u32, u32)> { None } -/// Extract a string from a scheme expression like `#"flute"`. -/// Returns `None` if the expression doesn't contain a quoted string. +/// The string of a Scheme string literal like `#"flute"`, escapes decoded; +/// `None` for any other Scheme value. pub(super) fn extract_scheme_string(text: &str) -> Option { - let inner = text.trim_start_matches('#').trim(); - if inner.starts_with('"') && inner.ends_with('"') && inner.len() >= 2 { - Some(inner[1..inner.len() - 1].to_string()) - } else { - None - } + super::text::scheme_string(text) } /// Build a Chord from a `chord` node (< ... >). @@ -841,15 +886,22 @@ pub(super) fn build_chord(state: &mut WalkState, chord_node: Node, dur: Duration let child = children[i]; if child.kind() == "symbol" { let sym = state.text(child); - if let Some((step, alter)) = parse_pitch_name(sym, state.language) { + let pitch = if state.drum_mode > 0 { + // A drum chord (``): absolute, and no reference for + // `\relative`. + state.drum_pitch(sym).inspect(|_| i += 1) + } else if let Some((step, alter)) = parse_pitch_name(sym, state.language) { i += 1; let octave_marks = consume_octave_marks(state, &children, &mut i); let acc_display = consume_accidental_marks(state, &children, &mut i); let mut pitch = state.resolve_pitch(step, alter, octave_marks); pitch.accidental = acc_display; - if first_pitch.is_none() { - first_pitch = Some(pitch); - } + first_pitch.get_or_insert(pitch); + Some(pitch) + } else { + None + }; + if let Some(pitch) = pitch { let mut note = Note::new(pitch, dur.clone()); // A member's own post-events: ``, ``. let attachments = consume_attachments(state, &children, &mut i); diff --git a/src/adapters/ly_to_ir/drums.rs b/src/adapters/ly_to_ir/drums.rs new file mode 100644 index 0000000..5a19e18 --- /dev/null +++ b/src/adapters/ly_to_ir/drums.rs @@ -0,0 +1,135 @@ +//! Drum-mode note names of LilyPond 2.26.0 and the pitch whose MIDI +//! key sounds each (General MIDI percussion): `ly/drumpitch-init.ly`. Generated: +//! `python3 scripts/ly_builtins.py lilypond v2.26.0`. + +/// (name, octave, step 0-6 from C, alteration in semitones), sorted by name. +pub(super) const DRUMS: &[(&str, i32, u8, i32)] = &[ + ("acousticbassdrum", 1, 6, 0), + ("acousticsnare", 2, 1, 0), + ("agh", 4, 4, 0), + ("agl", 4, 4, 1), + ("bassdrum", 2, 0, 0), + ("bd", 2, 0, 0), + ("bda", 1, 6, 0), + ("boh", 4, 0, 0), + ("bohm", 3, 6, 1), + ("boho", 4, 1, -2), + ("bol", 4, 0, 1), + ("bolm", 3, 6, 2), + ("bolo", 4, 1, -1), + ("cab", 4, 5, 0), + ("cabasa", 4, 5, 0), + ("cb", 3, 4, 1), + ("cgh", 4, 2, -1), + ("cghm", 4, 1, 0), + ("cgho", 4, 1, 1), + ("cgl", 4, 2, 0), + ("cglm", 4, 2, -2), + ("cglo", 4, 1, 2), + ("chinesecymbal", 3, 2, 0), + ("cl", 5, 1, 1), + ("claves", 5, 1, 1), + ("closedhihat", 2, 3, 1), + ("cowbell", 3, 4, 1), + ("crashcymbal", 3, 1, -1), + ("crashcymbala", 3, 0, 1), + ("crashcymbalb", 3, 5, 0), + ("cuim", 5, 3, 1), + ("cuio", 5, 4, 0), + ("cymc", 3, 1, -1), + ("cymca", 3, 0, 1), + ("cymcb", 3, 5, 0), + ("cymch", 3, 2, 0), + ("cymr", 3, 2, -1), + ("cymra", 3, 1, 1), + ("cymrb", 3, 6, 0), + ("cyms", 3, 4, 0), + ("electricsnare", 2, 2, 0), + ("gui", 5, 0, 2), + ("guil", 5, 1, 0), + ("guiro", 5, 0, 2), + ("guis", 5, 0, 1), + ("halfopenhihat", 2, 5, 1), + ("handclap", 2, 1, 1), + ("hc", 2, 1, 1), + ("hh", 2, 4, -1), + ("hhc", 2, 3, 1), + ("hhho", 2, 5, 1), + ("hho", 2, 5, 1), + ("hhp", 2, 4, 1), + ("hhs", 2, 4, 1), + ("hiagogo", 4, 4, 0), + ("hibongo", 4, 0, 0), + ("hiconga", 4, 2, -1), + ("highfloortom", 2, 4, 0), + ("hightom", 3, 1, 0), + ("hihat", 2, 4, -1), + ("himidtom", 3, 0, 0), + ("hisidestick", 1, 6, 2), + ("hitimbale", 4, 3, 0), + ("hiwoodblock", 5, 2, 0), + ("loagogo", 4, 4, 1), + ("lobongo", 4, 0, 1), + ("loconga", 4, 2, 0), + ("longguiro", 5, 1, 0), + ("longwhistle", 5, 0, 0), + ("losidestick", 2, 1, -1), + ("lotimbale", 4, 3, 1), + ("lowfloortom", 2, 3, 0), + ("lowmidtom", 2, 6, 0), + ("lowoodblock", 5, 3, 0), + ("lowtom", 2, 5, 0), + ("mar", 4, 5, 1), + ("maracas", 4, 5, 1), + ("mutecuica", 5, 3, 1), + ("mutehibongo", 3, 6, 1), + ("mutehiconga", 4, 1, 0), + ("mutelobongo", 3, 6, 2), + ("muteloconga", 4, 2, -2), + ("mutetriangle", 5, 4, 1), + ("opencuica", 5, 4, 0), + ("openhibongo", 4, 1, -2), + ("openhiconga", 4, 1, 1), + ("openhihat", 2, 5, 1), + ("openlobongo", 4, 1, -1), + ("openloconga", 4, 1, 2), + ("opentriangle", 5, 5, 0), + ("pedalhihat", 2, 4, 1), + ("rb", 3, 3, 0), + ("ridebell", 3, 3, 0), + ("ridecymbal", 3, 2, -1), + ("ridecymbala", 3, 1, 1), + ("ridecymbalb", 3, 6, 0), + ("shortguiro", 5, 0, 1), + ("shortwhistle", 4, 6, 0), + ("sidestick", 2, 0, 1), + ("sn", 2, 2, -2), + ("sna", 2, 1, 0), + ("snare", 2, 2, -2), + ("sne", 2, 2, 0), + ("splashcymbal", 3, 4, 0), + ("splashhihat", 2, 4, 1), + ("ss", 2, 0, 1), + ("ssh", 1, 6, 2), + ("ssl", 2, 1, -1), + ("tamb", 3, 3, 1), + ("tambourine", 3, 3, 1), + ("timh", 4, 3, 0), + ("timl", 4, 3, 1), + ("tomfh", 2, 4, 0), + ("tomfl", 2, 3, 0), + ("tomh", 3, 1, 0), + ("toml", 2, 5, 0), + ("tommh", 3, 0, 0), + ("tomml", 2, 6, 0), + ("tri", 5, 4, 2), + ("triangle", 5, 4, 2), + ("trim", 5, 4, 1), + ("trio", 5, 5, 0), + ("vibraslap", 3, 5, 1), + ("vibs", 3, 5, 1), + ("wbh", 5, 2, 0), + ("wbl", 5, 3, 0), + ("whl", 5, 0, 0), + ("whs", 4, 6, 0), +]; diff --git a/src/adapters/ly_to_ir/figured_bass.rs b/src/adapters/ly_to_ir/figured_bass.rs index d334cdd..8f84c2f 100644 --- a/src/adapters/ly_to_ir/figured_bass.rs +++ b/src/adapters/ly_to_ir/figured_bass.rs @@ -3,7 +3,7 @@ use tree_sitter::Node; use crate::ir::duration::Duration; use crate::ir::harmony::{Figure, FiguredBass}; -use super::state::WalkState; +use super::state::{WalkState, MAX_ELEMENTS}; use super::FiguredBassEntry; /// Divisions per quarter note used when computing figured bass offsets. @@ -57,8 +57,11 @@ pub(super) fn parse_figuremode_block(state: &WalkState, block: Node) -> Vec i32 { // Adapter // --------------------------------------------------------------------------- +/// Stack of the reader's walk thread: with at most `MAX_WALK_DEPTH` nested +/// blocks at up to about 20 KB of frames each (unoptimised builds), 64 MB +/// leaves room to spare; untouched stack pages cost no memory. +const WALK_STACK_BYTES: usize = 64 << 20; + +/// What reading LilyPond text produced. +#[derive(Debug, Clone)] +pub struct LyReading { + /// One score per movement: each `\score` block and each top-level music + /// expression, in order, as LilyPond makes a score of each. + pub scores: Vec, + /// Syntax errors and what the walk did not read, in source order. + pub diagnostics: Vec, + /// A warning for each movement after the first, which + /// [`into_first`](Self::into_first) drops. + later_movements: Vec, +} + +impl LyReading { + /// The first movement and the diagnostics, with a warning for each + /// movement dropped. + pub fn into_first(self) -> (Score, Vec) { + let mut diagnostics = self.diagnostics; + diagnostics.extend(self.later_movements); + tidy(&mut diagnostics); + let score = self.scores.into_iter().next().unwrap_or_else(|| { + let mut s = Score::new(); + s.children.push(ScoreChild::Part(Part::new("P1"))); + s + }); + (score, diagnostics) + } +} + +/// The text of a `.ly` file. LilyPond reads UTF-8 only, so other bytes are a +/// parse error, not an I/O error. +pub fn read_source(path: &Path) -> Result { + String::from_utf8(std::fs::read(path)?).map_err(|e| { + AdapterError::Parse(format!( + "{} is not UTF-8 text (invalid byte at offset {})", + path.display(), + e.utf8_error().valid_up_to() + )) + }) +} + +/// Check LilyPond text. By default only the syntax, from the tree (fast, no +/// reading); with `semantic`, everything a reading reports too. Input the +/// reader refuses as too large is an error diagnostic here, not an `Err`. +pub fn check(text: &str, semantic: bool) -> Vec { + let too_large = |message: String| Diagnostic::whole(Severity::Error, "too-large", message); + if semantic { + return match LyToIrAdapter::new().read_str(text) { + Ok(reading) => reading.diagnostics, + Err(e) => { + let mut out = check(text, false); + out.push(too_large(e.to_string())); + tidy(&mut out); + out + } + }; + } + let parsed = LilyPondParser::new().and_then(|mut p| p.parse(text)); + match parsed { + Ok(tree) => syntax::syntax_diagnostics(&tree, text), + Err(e) => vec![too_large(e.to_string())], + } +} + +pub use text::HeaderField; + +/// Every `\header` field of LilyPond text, in source order, from the syntax +/// tree alone (nothing is read): its key, its value as text (strings decoded, +/// `\markup` as plain text, `#"…"` as the string), the byte range of `key = +/// value` and the `\score` block it is in, by order, if any. Fields with other +/// values (`##f`, Scheme) are left out. +pub fn header_fields(text: &str) -> Vec { + match LilyPondParser::new().and_then(|mut p| p.parse(text)) { + Ok(tree) => text::header_fields_of(text, tree.root_node()), + Err(_) => Vec::new(), + } +} + /// LilyPond → IR adapter. /// /// Parses LilyPond source text using tree-sitter and produces an IR `Score`. @@ -192,57 +280,85 @@ impl LyToIrAdapter { self } - /// Parse LilyPond source text into one or more IR Scores (one per `\score` block). - fn parse_source_multi(&self, source: &str) -> Result> { + /// Read LilyPond text: every movement, and the diagnostics. + /// + /// The walk recurses once per nested music block, so it runs on a thread + /// with a stack of its own: how deep a score may nest must not depend on + /// the caller's thread (Rust test threads and many Python threads get 2 MB + /// or less). A panic on that thread resumes on the caller's. + pub fn read_str(&self, text: &str) -> Result { + std::thread::scope(|scope| { + std::thread::Builder::new() + .name("lytk-ly-reader".to_string()) + .stack_size(WALK_STACK_BYTES) + .spawn_scoped(scope, || self.read_here(text))? + .join() + .unwrap_or_else(|payload| std::panic::resume_unwind(payload)) + }) + } + + /// Read a LilyPond file (see [`read_source`]). + pub fn read_file(&self, path: &Path) -> Result { + self.read_str(&read_source(path)?) + } + + fn read_here(&self, source: &str) -> Result { let mut parser = LilyPondParser::new().map_err(|e| AdapterError::Parse(e.to_string()))?; let tree = parser .parse(source) .map_err(|e| AdapterError::Parse(e.to_string()))?; - let root = tree.root_node(); - if root.has_error() { - // Still attempt to extract what we can; tree-sitter is error-tolerant - } + let mut diagnostics = syntax::syntax_diagnostics(&tree, source); let mut state = state::WalkState::new(source); state.root = Some(root); state.language = self.language; walk::walk_program(&mut state, root); - - // `\score` blocks were assembled as they closed (one per movement). - if !state.completed_scores.is_empty() { - return Ok(state.completed_scores); + let refused = |reason: String| { + AdapterError::Unsupported(format!("{reason}; lytk refuses input this large")) + }; + if let Some(reason) = state.over_limit.take() { + return Err(refused(reason)); } - // Otherwise the file's top-level music is one implicit score. - let mut score = assemble_score(&mut state).unwrap_or_default(); - if score.children.is_empty() { + // Every movement is assembled; a file without music is one empty one. + let mut scores = std::mem::take(&mut state.completed_scores); + if scores.is_empty() { + let mut score = Score::default(); score.children.push(ScoreChild::Part(Part::new("P1"))); + scores.push(score); } - Ok(vec![score]) + // The book's header, last assignment first, where a movement leaves a + // field unset. + for score in &mut scores { + for (key, value) in state.book_header.iter().rev() { + walk::set_header_field(&mut score.metadata, key, value.clone(), false); + } + } + diagnostics.append(&mut state.diagnostics); + tidy(&mut diagnostics); + Ok(LyReading { + scores, + diagnostics, + later_movements: state.later_movements, + }) } /// Parse LilyPond source text into an IR Score. /// If there are multiple `\score` blocks, returns only the first one. fn parse_source(&self, source: &str) -> Result { - let scores = self.parse_source_multi(source)?; - Ok(scores.into_iter().next().unwrap_or_else(|| { - let mut s = Score::new(); - s.children.push(ScoreChild::Part(Part::new("P1"))); - s - })) + Ok(self.read_str(source)?.into_first().0) } /// Parse LilyPond source into multiple scores (one per movement). pub fn convert_file_multi(&self, path: &Path) -> Result> { - let source = std::fs::read_to_string(path)?; - self.parse_source_multi(&source) + Ok(self.read_file(path)?.scores) } /// Parse LilyPond source string into multiple scores (one per movement). pub fn convert_str_multi(&self, text: &str) -> Result> { - self.parse_source_multi(text) + Ok(self.read_str(text)?.scores) } } @@ -299,6 +415,13 @@ fn assemble_score(state: &mut state::WalkState) -> Option { } let grid = Grid::build(parts.iter().map(|pb| &pb.tl)); + if grid.bars.len() >= crate::ir::timeline::MAX_BARS { + state.refuse(format!( + "the music reaches {} bars", + crate::ir::timeline::MAX_BARS + )); + return None; + } let mut built: Vec<(u32, Part)> = parts .into_iter() .map(|pb| { @@ -369,8 +492,7 @@ impl Default for LyToIrAdapter { impl ToIrAdapter for LyToIrAdapter { fn convert_file(&self, path: &Path) -> Result { - let source = std::fs::read_to_string(path)?; - self.parse_source(&source) + self.parse_source(&read_source(path)?) } fn convert_str(&self, text: &str) -> Result { diff --git a/src/adapters/ly_to_ir/modifiers.rs b/src/adapters/ly_to_ir/modifiers.rs index 4fadfce..39f975a 100644 --- a/src/adapters/ly_to_ir/modifiers.rs +++ b/src/adapters/ly_to_ir/modifiers.rs @@ -94,13 +94,16 @@ pub(super) fn consume_repeat(state: &mut WalkState, children: &[Node], mut i: us }; // Parse repeat count - let repeat_count: u8 = if i < children.len() && children[i].kind() == "unsigned_integer" { - let n = state.text(children[i]).parse().unwrap_or(2); + // Past `u64` a count is unreadable: it counts as the most there can be, + // which the unfold loop's budget then refuses. + let count: u64 = if i < children.len() && children[i].kind() == "unsigned_integer" { + let n = state.text(children[i]).parse().unwrap_or(u64::MAX); i += 1; n } else { 2 }; + let repeat_count = u8::try_from(count).unwrap_or(u8::MAX); // For unfold repeats, walk the body block `repeat_count` times — the music // is repeated literally (was previously walked only once, dropping N-1 @@ -108,7 +111,17 @@ pub(super) fn consume_repeat(state: &mut WalkState, children: &[Node], mut i: us if repeat_type == "unfold" { if i < children.len() && children[i].kind() == "expression_block" { let body = children[i]; - for _ in 0..repeat_count.max(1) { + // LilyPond makes the written music relative once and then copies + // it: every pass starts from the same reference pitch, so copies + // are identical instead of climbing an octave step per pass. + // Each pass counts against the reading's budget, so an unfold of + // an empty body cannot spin for billions of passes either. + let reference = state.prev_pitch; + for _ in 0..count.max(1) { + if !state.spend(1) { + break; + } + state.prev_pitch = reference; walk_music_block(state, body); } return i + 1; diff --git a/src/adapters/ly_to_ir/music.rs b/src/adapters/ly_to_ir/music.rs index 0dfd960..b8f87be 100644 --- a/src/adapters/ly_to_ir/music.rs +++ b/src/adapters/ly_to_ir/music.rs @@ -19,7 +19,7 @@ use super::consume::{ build_chord, consume_accidental_marks, consume_attachments, consume_duration, consume_duration_scale, consume_mark, consume_octave_marks, consume_override, consume_tempo, consume_tremolo, extract_scheme_string, extract_string_value, is_dynamic_name, parse_fraction, - parse_grace_block, parse_ly_make_moment, parse_paper_block, punct_text, + parse_grace_block, parse_ly_make_moment, parse_paper_block, previous, punct_text, skip_markup, }; use super::merge::apply_tuplet_display; use super::modifiers::{consume_relative, consume_repeat, consume_transpose}; @@ -60,8 +60,100 @@ fn parse_skip_of_length(scheme_text: &str) -> Option<&str> { (!var.is_empty()).then_some(var) } -/// Walk an `expression_block` `{ ... }` containing music. +/// Walk an `expression_block` `{ ... }` containing music, in the input mode +/// the command before it sets: `\drummode` reads drum names, `\drums` is a +/// drum staff of them; `\chords` and `\figures` (`\figuremode`) are chord +/// names and figured bass, not notes; lyrics are not notes either. pub(super) fn walk_music_block(state: &mut WalkState, block: Node) { + // Past a bound, re-walks (unfolds, variables) end here. + if state.stopped() || !state.enter_block() { + return; + } + let mut prev = block.prev_sibling(); + while prev.is_some_and(|p| p.kind() == "comment") { + prev = prev.and_then(|p| p.prev_sibling()); + } + let mode = prev + .filter(|p| p.kind() == "escaped_word") + .map(|p| state.text(p)); + if let Some(octaves) = fixed_octaves(state, block) { + // `\fixed c' { … }`: absolute pitches, an octave up per mark, even + // inside `\relative`. + let saved = (state.in_relative, state.relative_depth, state.fixed_octaves); + state.in_relative = false; + state.relative_depth = 0; + state.fixed_octaves = octaves; + walk_block_contents(state, block); + (state.in_relative, state.relative_depth, state.fixed_octaves) = saved; + state.walk_depth -= 1; + return; + } + match mode { + Some("\\drums") => { + state.new_part("DrumStaff", ""); + add_percussion_clef(state); + state.drum_mode += 1; + walk_block_contents(state, block); + state.drum_mode -= 1; + } + Some("\\drummode") => { + state.drum_mode += 1; + walk_block_contents(state, block); + state.drum_mode -= 1; + } + // A chord-mode variable, walked again for its notes, stays notes. + Some("\\chords") if state.quiet == 0 => { + let entries = super::chord_mode::parse_chordmode_block(state, block); + state.pending_harmonies.extend(entries); + } + Some("\\figures" | "\\figuremode") => { + let entries = super::figured_bass::parse_figuremode_block(state, block); + state.place_figures(entries); + } + Some("\\lyrics" | "\\lyricmode") => {} + // Chords in a staff are their notes, which the walk reads as roots + // only: no warnings about the chord names' suffixes. + Some("\\chordmode" | "\\chords") => { + state.quiet += 1; + walk_block_contents(state, block); + state.quiet -= 1; + } + _ => walk_block_contents(state, block), + } + state.walk_depth -= 1; +} + +/// The octaves of `\fixed ` right before `block`, if it follows one: +/// the pitch's octave marks. +fn fixed_octaves(state: &WalkState, block: Node) -> Option { + let mut marks = 0; + let mut at = block.prev_sibling(); + while let Some(n) = at.filter(|n| n.kind() == "punctuation") { + marks += match state.text(n) { + "'" => 1, + "," => -1, + _ => return None, + }; + at = n.prev_sibling(); + } + let pitch = at.filter(|n| n.kind() == "symbol")?; + let fixed = pitch.prev_sibling()?; + (state.text(fixed) == "\\fixed").then_some(marks) +} + +/// A percussion clef where the current part is (a drum staff). +pub(super) fn add_percussion_clef(state: &mut WalkState) { + if let Some((sign, line, octave_change)) = parse_clef_name("percussion") { + let clef = Clef { + sign, + line, + octave_change, + }; + state.add_event(Event::Clef(1, clef)); + } +} + +fn walk_block_contents(state: &mut WalkState, block: Node) { let mut cursor = block.walk(); let children: Vec = block.children(&mut cursor).collect(); let mut i = 0; @@ -151,7 +243,13 @@ pub(super) fn walk_music_block(state: &mut WalkState, block: Node) { /// Handle a symbol node (pitch name, r, R, s, etc.). /// Returns the next index to process. fn handle_symbol(state: &mut WalkState, children: &[Node], i: usize, sym: &str) -> usize { + let (node, prev) = (children[i], previous(children, i)); let mut i = i + 1; + if prev.is_some_and(|p| state.text(p) == "\\fixed") { + // `\fixed`'s pitch, not a note: the block after it reads it. + consume_octave_marks(state, children, &mut i); + return i; + } match sym { "r" => { @@ -231,6 +329,9 @@ fn handle_symbol(state: &mut WalkState, children: &[Node], i: usize, sym: &str) // Fractional multiplier: scale the duration dur.base *= frac; for _ in 0..repeat_count { + if state.stopped() { + break; + } let mut rest = Rest::new(dur.clone()); rest.is_spacer = true; apply_rest_attachments(&mut rest, &attachments); @@ -241,6 +342,9 @@ fn handle_symbol(state: &mut WalkState, children: &[Node], i: usize, sym: &str) // Integer multiplier: push N spacer rests let count = *frac.numer() as u32; for _ in 0..count { + if state.stopped() { + break; + } let mut rest = Rest::new(dur.clone()); rest.is_spacer = true; state.push_voice_element(VoiceElement::Rest(rest)); @@ -302,6 +406,27 @@ fn handle_symbol(state: &mut WalkState, children: &[Node], i: usize, sym: &str) state.push_voice_element(VoiceElement::Chord(chord)); } } + _ if state.drum_mode > 0 => match state.drum_pitch(sym) { + // A drum note: absolute, and outside `\relative` and `\transpose`. + Some(pitch) => { + let mut dur = consume_duration(state, children, &mut i); + if let Some(scale) = consume_duration_scale(state, children, &mut i) { + dur.base *= scale; + state.last_duration = dur.clone(); + } + let tremolo = consume_tremolo(state, children, &mut i, &dur); + let attachments = consume_attachments(state, children, &mut i); + let mut note = Note::new(pitch, dur); + note.tremolo_marks = tremolo; + apply_note_attachments(state, &mut note, &attachments); + state.push_voice_element(note_or_pitched_rest(note, &attachments)); + } + None => state.warn( + node, + "unrecognized-token", + format!("`{sym}` is not a drum name"), + ), + }, _ => { // Try as pitch name if let Some((step, alter)) = parse_pitch_name(sym, state.language) { @@ -332,8 +457,19 @@ fn handle_symbol(state: &mut WalkState, children: &[Node], i: usize, sym: &str) } apply_note_attachments(state, &mut note, &attachments); state.push_voice_element(note_or_pitched_rest(note, &attachments)); + } else if !prev.is_some_and(|p| { + matches!(p.kind(), "escaped_word" | "embedded_scheme" | "string") + || (p.kind() == "punctuation" && state.text(p) == "=") + }) { + // Not a command's argument (a context name, a `\repeat` + // type…) either: a word the walk does not read. + let language = state.language.as_str(); + state.warn( + node, + "unrecognized-token", + format!("`{sym}` is not a {language} note name"), + ); } - // If not a pitch name, ignore (could be a context name etc.) } } i @@ -385,10 +521,38 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: } i += 2; } - if let Some(frac_node) = children.get(i) { + if let Some(&frac_node) = children.get(i) { if frac_node.kind() == "fraction" { - let frac_text = state.text(*frac_node); - if let Some((num, den)) = parse_fraction(frac_text) { + let frac_text = state.text(frac_node); + // A zero term is no time signature; a denominator the IR's + // `u8` cannot hold (`\time 3/256`) is dropped: truncated, + // it would become 0. + let parsed = match parse_fraction(frac_text) { + Some((num, den)) if num == 0 || den == 0 => { + // LilyPond, too, only warns and ignores it. + state.warn( + frac_node, + "unsupported-value", + format!("`\\time {frac_text}` has a zero term: ignored"), + ); + None + } + parsed => { + let parsed = parsed.and_then(|(n, d)| Some((n, u8::try_from(d).ok()?))); + if parsed.is_none() { + state.warn( + frac_node, + "unsupported-value", + format!( + "`\\time {frac_text}`: lytk reads denominators up to \ + 255; dropped" + ), + ); + } + parsed + } + }; + if let Some((num, den)) = parsed { let beats = if extra_beats.is_empty() { num.to_string() } else { @@ -401,7 +565,7 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: }; let ts = TimeSignature { beats, - beat_type: den as u8, + beat_type: den, symbol: None, }; state.add_event(Event::Time(ts)); @@ -496,17 +660,46 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: } "\\tuplet" | "\\times" => { // \tuplet actual/normal { notes } OR \times normal/actual { notes } - if let Some(frac_node) = children.get(i) { + if let Some(&frac_node) = children.get(i) { if frac_node.kind() == "fraction" { - let frac_text = state.text(*frac_node); + let frac_text = state.text(frac_node); if let Some((num, denom)) = frac_text.split_once('/') { - let n: u8 = num.parse().unwrap_or(1); - let d: u8 = denom.parse().unwrap_or(1); - let (actual, normal) = if text == "\\tuplet" { - (n, d) - } else { - (d, n) // \times has reversed fraction + // A ratio with a 0 or a term beyond the IR's `u8` + // (`\tuplet 0/2`, `\times 2/0`, `\tuplet 300/2`) scales + // nothing: its music is read unscaled. + let ratio = match (num.parse::(), denom.parse::()) { + (Ok(0), _) | (_, Ok(0)) => { + state.error( + frac_node, + "invalid-ratio", + format!("`{text} {frac_text}` has a zero term: read unscaled"), + ); + None + } + (n, d) => { + let term = + |t: Result| t.ok().and_then(|t| u8::try_from(t).ok()); + let ratio = term(n).zip(term(d)); + if ratio.is_none() { + state.warn( + frac_node, + "unsupported-value", + format!( + "`{text} {frac_text}`: lytk reads tuplet terms up to \ + 255; read unscaled" + ), + ); + } + ratio + } }; + let ratio = ratio.map(|(n, d)| { + if text == "\\tuplet" { + (n, d) + } else { + (d, n) // \times has reversed fraction + } + }); i += 1; // Optional group-duration argument, e.g. `\tuplet 3/2 4 { … }` // (the `4` tells LilyPond the span of each tuplet group for @@ -526,20 +719,24 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: } if let Some(block) = children.get(i) { if block.kind() == "expression_block" { - // Push tuplet ratio so notes created inside get - // the scaling applied immediately (for correct - // measure duration tracking). - state.tuplet_stack.push((actual, normal)); - let before = state.current_voice.len(); - walk_music_block(state, *block); - let after = state.current_voice.len(); - state.tuplet_stack.pop(); - // Apply tuplet display markers (start/stop brackets) - if after > before { - apply_tuplet_display( - &mut state.current_voice[before..after], - actual, - ); + if let Some((actual, normal)) = ratio { + // Push tuplet ratio so notes created inside get + // the scaling applied immediately (for correct + // measure duration tracking). + state.tuplet_stack.push((actual, normal)); + let before = state.current_voice.len(); + walk_music_block(state, *block); + let after = state.current_voice.len(); + state.tuplet_stack.pop(); + // Apply tuplet display markers (start/stop brackets) + if after > before { + apply_tuplet_display( + &mut state.current_voice[before..after], + actual, + ); + } + } else { + walk_music_block(state, *block); } i += 1; } @@ -596,19 +793,9 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: let scheme_text = state.text(*scheme_node); // Parse #N or #-N from the embedded scheme let num_str = scheme_text.trim_start_matches('#'); - if let Ok(n) = num_str.parse::() { - let (shift_type, size) = if n > 0 { - ("up", (n * 8) as i8) - } else if n < 0 { - ("down", (n.abs() * 8) as i8) - } else { - ("stop", 0i8) - }; + if let Ok(n) = num_str.parse::() { let dir = Direction { - octave_shift: Some(OctaveShift { - shift_type: shift_type.to_string(), - size, - }), + octave_shift: Some(OctaveShift::from_octaves(n)), ..Default::default() }; state.add_event(Event::direction(dir)); @@ -815,14 +1002,30 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: // Handle \set Staff.midiInstrument = #"flute" let scheme_text = state.text(*val_node); if prop_text.contains("measureLength") { - if let Some((num, den)) = parse_ly_make_moment(scheme_text) { - let len = Frac::new(num as i64, den as i64); - state.add_event(Event::MeasureLength(len)); + match parse_ly_make_moment(scheme_text) { + Some((_, 0)) => state.error( + *val_node, + "invalid-ratio", + "a measure length with a zero denominator: dropped" + .to_string(), + ), + Some((0, _)) => state.warn( + *val_node, + "unsupported-value", + "a measure length of zero: dropped".to_string(), + ), + Some((num, den)) => { + let len = Frac::new(num as i64, den as i64); + state.add_event(Event::MeasureLength(len)); + } + None => {} } } else if let Some(s) = extract_scheme_string(scheme_text) { apply_set_property(state, &prop_text, &s); } i += 1; + } else if matches!(state.text(*val_node), "\\markup" | "\\markuplist") { + i = skip_markup(state, children, i + 1); } else { // Skip unknown value types (scheme booleans, etc.) i += 1; @@ -845,6 +1048,9 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: Some(frac) => { let count = *frac.numer() as u32; for _ in 0..count { + if state.stopped() { + break; + } let mut rest = Rest::new(dur.clone()); rest.is_spacer = true; state.push_voice_element(VoiceElement::Rest(rest)); @@ -879,11 +1085,25 @@ fn handle_escaped_word(state: &mut WalkState, children: &[Node], i: usize, text: // walk_music_block level, but if tree-sitter doesn't wrap it as // named_context, skip it here. } + "\\markup" | "\\markuplist" => { + // Text standing on its own (as a `\tempo` or `\set` value): not music. + i = skip_markup(state, children, i); + } + "\\language" => { + if let Some(&next) = children.get(i).filter(|n| n.kind() == "string") { + state.set_language(next); + i += 1; + } + } _ => { // Unknown escaped word — may be a variable reference or dynamic let var_name = text.trim_start_matches('\\'); - if !state.resolve_variable(var_name) && is_dynamic_name(text) { - attach_dynamic(state, text); + if !state.resolve_variable(var_name) { + if is_dynamic_name(text) { + attach_dynamic(state, text); + } else { + state.unread_command(children[i - 1], var_name); + } } } } diff --git a/src/adapters/ly_to_ir/state.rs b/src/adapters/ly_to_ir/state.rs index e832712..9b691f8 100644 --- a/src/adapters/ly_to_ir/state.rs +++ b/src/adapters/ly_to_ir/state.rs @@ -1,8 +1,10 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use num::rational::Ratio; +use num::CheckedAdd; use tree_sitter::Node; +use crate::diagnostics::{Columns, Diagnostic, Severity}; use crate::ir::articulation::{BeamEvent, LyricSyllable}; use crate::ir::duration::{Duration, Frac}; use crate::ir::language::{PitchLanguage, PitchMode}; @@ -43,13 +45,51 @@ pub(super) enum VarDef { main_lane: u8, voices: Vec, /// Byte range of the `{ … }` block that defines it, so a reference - /// inside `\relative` can read it again there (see `resolve_variable`). - block: Option<(usize, usize)>, + /// inside `\relative` can read it again there (see `resolve_variable`), + /// in the pitch language of its definition. + block: Option<(usize, usize, PitchLanguage)>, }, /// Variable contained `\figuremode { ... }` — stores flat stream of entries. FiguredBass(Vec), } +/// Most voice elements (notes, rests, chords) one reading may generate, and +/// the furthest a voice may reach, in whole notes. Real scores stay orders of +/// magnitude below; the bounds keep `s1*4000000000`, nested `\repeat unfold` +/// and self-doubling variables from hanging or exhausting memory. Past either +/// one the reading stops and fails. +pub(super) const MAX_ELEMENTS: u64 = 500_000; +pub(super) const MAX_WHOLE_NOTES: i64 = 100_000; +/// Octaves a pitch may lie in (middle C is octave 4; MIDI reaches -1..=9). +/// Far wider than music: LilyPond's own regression tests climb to octave 22 +/// with repeated relative scales. Emitters write one mark per octave, so the +/// bound caps each note's output at about 127 marks. +pub(super) const PITCH_OCTAVES: std::ops::RangeInclusive = -128..=127; +/// LilyPond's language files (its `ly/` directory), and the language each +/// sets: `arabic.ly` uses Italian note names, with accidentals of its own. +const LANGUAGE_FILES: [(&str, PitchLanguage); 12] = [ + ("arabic", PitchLanguage::Italiano), + ("catalan", PitchLanguage::Catalan), + ("deutsch", PitchLanguage::Deutsch), + ("english", PitchLanguage::English), + ("espanol", PitchLanguage::Espanol), + ("italiano", PitchLanguage::Italiano), + ("nederlands", PitchLanguage::Nederlands), + ("norsk", PitchLanguage::Norsk), + ("portugues", PitchLanguage::Portugues), + ("suomi", PitchLanguage::Suomi), + ("svenska", PitchLanguage::Svenska), + ("vlaams", PitchLanguage::Vlaams), +]; +/// LilyPond's files of pitch names lytk cannot read (with quarter and +/// smaller tones of their own). +const OTHER_PITCH_NAMES: [&str; 5] = ["bagpipe", "hel-arabic", "makam", "persian", "turkish-makam"]; + +/// Deepest the walk may recurse: music blocks inside music blocks, counting +/// the variables read again inside `\relative`, which the syntax tree's own +/// depth bound does not see. +pub(super) const MAX_WALK_DEPTH: u32 = crate::parser::MAX_NESTING_DEPTH as u32; + /// State accumulated while walking tree-sitter nodes. pub(super) struct WalkState<'src> { pub(super) source: &'src str, @@ -57,6 +97,14 @@ pub(super) struct WalkState<'src> { pub(super) root: Option>, /// Nesting of variables being read again, against self-reference. pub(super) var_depth: u8, + /// Voice elements generated so far, against [`MAX_ELEMENTS`]. + pub(super) generated: u64, + /// Why the reading stopped, once it went past a bound. + pub(super) over_limit: Option, + /// Music blocks being walked, one inside the other (see [`MAX_WALK_DEPTH`]). + pub(super) walk_depth: u32, + /// Variables being read again inside `\relative`, innermost last. + pub(super) rewalking: Vec, /// Nesting of `\relative { … }` blocks being walked. Unlike `in_relative` /// (which can stay set after a `\relative` for the music that follows), /// this is exact: it is what decides whether a variable is read again. @@ -148,6 +196,27 @@ pub(super) struct WalkState<'src> { /// The context just read was `\context X` rather than `\new X`: it /// re-enters an existing X instead of creating one. pub(super) context_reentry: std::cell::Cell, + + /// What the walk found wrong, or did not read. + pub(super) diagnostics: Vec, + columns: Columns, + /// Above 0 while music already walked once is walked for another purpose + /// (a chord-mode block read as notes): the first walk reported it. + pub(super) quiet: u32, + /// Names assigned at the top level (`name = …`), whatever their value. + pub(super) assigned: HashSet, + /// The top-level music expression being read, if any: LilyPond makes a + /// score of each, so it is a movement of its own. + pub(super) open_movement: Option>, + /// Octaves `\fixed` adds to absolute pitches (`\fixed c' { c }` is c'). + pub(super) fixed_octaves: i32, + /// The `\score` blocks after the first, which a single-score reading drops. + pub(super) later_movements: Vec, + /// Fields of the top-level `\header` blocks, in order: they apply to + /// every movement that does not set them itself. + pub(super) book_header: Vec<(String, String)>, + /// Above 0 inside `\drummode` music: words are drum names. + pub(super) drum_mode: u32, } impl<'src> WalkState<'src> { @@ -180,6 +249,10 @@ impl<'src> WalkState<'src> { in_relative: false, root: None, var_depth: 0, + generated: 0, + over_limit: None, + walk_depth: 0, + rewalking: Vec::new(), relative_depth: 0, pending_arpeggio_type: None, pending_glissando_style: None, @@ -194,11 +267,20 @@ impl<'src> WalkState<'src> { transpose_stack: Vec::new(), current_voice_number: 1, context_reentry: std::cell::Cell::new(false), + diagnostics: Vec::new(), + columns: Columns::default(), + quiet: 0, + assigned: HashSet::new(), + open_movement: None, + fixed_octaves: 0, + later_movements: Vec::new(), + book_header: Vec::new(), + drum_mode: 0, } } /// Get the text content of a node. - pub(super) fn text(&self, node: Node) -> &str { + pub(super) fn text(&self, node: Node) -> &'src str { node.utf8_text(self.source.as_bytes()).unwrap_or("") } @@ -229,8 +311,163 @@ impl<'src> WalkState<'src> { self.ensure_build().tl.add(pos, ev); } + /// Report a finding about `node`. + pub(super) fn report( + &mut self, + node: Node, + severity: Severity, + code: &'static str, + message: String, + ) { + if self.quiet == 0 { + let (source, end) = (self.source, node.end_byte()); + let d = Diagnostic::counted( + &mut self.columns, + source, + node, + end, + severity, + code, + message, + ); + self.diagnostics.push(d); + } + } + + pub(super) fn warn(&mut self, node: Node, code: &'static str, message: String) { + self.report(node, Severity::Warning, code, message); + } + + pub(super) fn error(&mut self, node: Node, code: &'static str, message: String) { + self.report(node, Severity::Error, code, message); + } + + /// `\language "name"` (`node` is the string): the pitch names from here + /// on. An unknown name keeps the current language, and warns. + pub(super) fn set_language(&mut self, node: Node) { + let name = super::text::string_value(self.source, node); + match PitchLanguage::from_str_loose(&name) { + Some(lang) => self.language = lang, + None => { + let current = self.language.as_str(); + self.warn( + node, + "unknown-language", + format!("unknown pitch language `{name}`: {current} stays"), + ); + } + } + } + + /// `\include` (at `node`): LilyPond's language files set the language as + /// `\language` does; any other file is not followed. + fn include(&mut self, node: Node) { + let file = node.next_sibling().filter(|n| n.kind() == "string"); + let path = file.map(|n| super::text::string_value(self.source, n)); + let stem = path + .as_deref() + .and_then(|p| p.rsplit(['/', '\\']).next()) + .and_then(|f| f.strip_suffix(".ly")); + if let Some(&(_, lang)) = LANGUAGE_FILES.iter().find(|(f, _)| Some(*f) == stem) { + self.language = lang; + } else if let Some(stem) = stem.filter(|s| OTHER_PITCH_NAMES.contains(s)) { + self.warn( + node, + "unknown-language", + format!("the pitch names of `{stem}.ly` are not read"), + ); + } else { + let shown = file + .map(|n| format!(" {}", self.text(n))) + .unwrap_or_default(); + self.warn( + node, + "ignored-include", + format!("`\\include{shown}` is not followed: flatten the file first"), + ); + } + } + + /// A command at `node` (`\name`) that the walk does not read: warn, + /// unless LilyPond or the file defines it. + pub(super) fn unread_command(&mut self, node: Node, name: &str) { + if name == "include" { + self.include(node); + } else if !self.assigned.contains(name) + && super::builtins::BUILTINS.binary_search(&name).is_err() + { + self.warn( + node, + "unknown-command", + format!("unknown command `\\{name}`"), + ); + } + } + + /// Stop the reading: it went past a bound. The first reason is kept. + pub(super) fn refuse(&mut self, reason: String) { + self.over_limit.get_or_insert(reason); + } + + /// Whether the reading stopped at a bound; walkers return early then. + pub(super) fn stopped(&self) -> bool { + self.over_limit.is_some() + } + + /// Enter one more nested music block: false (and the reading stops) + /// past [`MAX_WALK_DEPTH`]. The caller decrements `walk_depth` on exit. + pub(super) fn enter_block(&mut self) -> bool { + if self.walk_depth >= MAX_WALK_DEPTH { + self.refuse(format!( + "the music nests deeper than {MAX_WALK_DEPTH} levels" + )); + return false; + } + self.walk_depth += 1; + true + } + + /// Where music of length `len` placed at `at` ends, if within the bounds + /// (the reading stops otherwise). + pub(super) fn end_within_bounds(&mut self, at: Frac, len: Frac) -> Option { + match at.checked_add(&len) { + Some(end) if end <= Frac::from_integer(MAX_WHOLE_NOTES) => Some(end), + Some(_) => { + self.refuse(format!( + "the music is longer than {MAX_WHOLE_NOTES} whole notes" + )); + None + } + None => { + self.refuse( + "note positions overflow: the durations' denominators are too large" + .to_string(), + ); + None + } + } + } + + /// Count `n` more generated elements: false once past [`MAX_ELEMENTS`]. + pub(super) fn spend(&mut self, n: u64) -> bool { + if self.stopped() { + return false; + } + self.generated = self.generated.saturating_add(n); + if self.generated > MAX_ELEMENTS { + self.refuse(format!( + "the music expands to more than {MAX_ELEMENTS} notes, rests and chords" + )); + return false; + } + true + } + /// Push a voice element at the current position. pub(super) fn push_voice_element(&mut self, mut elem: VoiceElement) { + if !self.spend(1) { + return; + } // Apply active tuplet ratio to the element's duration. For nested // tuplets the effective scaling is the product of every enclosing // ratio, not just the innermost — so fold the whole stack. @@ -312,7 +549,10 @@ impl<'src> WalkState<'src> { // attaches at the note rather than after its duration. self.last_element_onset = self.pos; // Grace notes take no time. - self.pos += elem.metric_duration(); + let Some(end) = self.end_within_bounds(self.pos, elem.metric_duration()) else { + return; + }; + self.pos = end; self.current_voice.push(elem); } @@ -384,6 +624,48 @@ impl<'src> WalkState<'src> { } } + /// In drum mode, the pitch whose MIDI key sounds drum `name` (`bd`, + /// `snare`, …), as MusicXML keeps unpitched notes on a percussion staff. + pub(super) fn drum_pitch(&self, name: &str) -> Option { + if self.drum_mode == 0 { + return None; + } + let drums = super::drums::DRUMS; + let &(_, octave, step, alter) = drums + .binary_search_by(|(n, ..)| n.cmp(&name)) + .ok() + .map(|k| &drums[k])?; + let step = [ + PitchStep::C, + PitchStep::D, + PitchStep::E, + PitchStep::F, + PitchStep::G, + PitchStep::A, + PitchStep::B, + ][step as usize]; + Some(Pitch::with_alter(step, Ratio::from_integer(alter), octave)) + } + + /// Figured bass from the current position on (a `\figuremode` block or + /// variable): events, taking no time in the voice. + pub(super) fn place_figures(&mut self, entries: Vec) { + if !self.spend(entries.len() as u64) { + return; + } + let mut at = self.pos; + for entry in entries { + match entry { + FiguredBassEntry::Figure(fb) => { + let d = fb.duration.actual_duration(); + self.add_event_at(at, Event::FiguredBass(fb)); + at += d; + } + FiguredBassEntry::Skip(dur) => at += dur.actual_duration(), + } + } + } + /// Resolve a variable reference at the current position. pub(super) fn resolve_variable(&mut self, name: &str) -> bool { // `\relative` applies to a variable's music where it is used: LilyPond @@ -391,22 +673,34 @@ impl<'src> WalkState<'src> { // written with plain pitches (`cadenza = { fis2 … }` used in // `\relative c'' { \cadenza }`) is therefore read again here, in the // relative context, rather than spliced as read at its definition. - if self.relative_depth > 0 && self.var_depth < 16 { + // A variable being read again is not read again inside itself (`a = + // { \a \a }` refers to the previous `a`): its captured music is used. + if self.relative_depth > 0 + && self.var_depth < 16 + && !self.rewalking.iter().any(|n| n == name) + { if let Some(VarDef::Music { - block: Some((start, end)), + block: Some((start, end, language)), .. }) = self.definitions.get(name) { - let (start, end) = (*start, *end); + let (start, end, language) = (*start, *end, *language); let node = self .root .and_then(|r| r.descendant_for_byte_range(start, end)); let node = std::iter::successors(node, |n| n.parent()).find(|n| { n.kind() == "expression_block" && (n.start_byte(), n.end_byte()) == (start, end) }); - if let Some(block) = node { + // A block that names its own variable (`a = { \a \a }`) meant + // the previous `a`, which is gone: use the music captured at + // the definition instead of reading the block again. + if let Some(block) = node.filter(|b| !mentions(*b, self.source, name)) { self.var_depth += 1; + self.rewalking.push(name.to_string()); + let outer = std::mem::replace(&mut self.language, language); super::music::walk_music_block(self, block); + self.language = outer; + self.rewalking.pop(); self.var_depth -= 1; return true; } @@ -414,10 +708,21 @@ impl<'src> WalkState<'src> { } // Positioned music is spliced straight from the definition, without // copying it first. - if let Some(VarDef::Music { len, main_lane, .. }) = self.definitions.get(name) { - let (len, main_lane) = (*len, *main_lane); + // Every copy counts against the budget: definitions that double the + // previous one (`b = { \a \a }`, `c = { \b \b }`, …) grow as 2^n. + if let Some(VarDef::Music { + len, main_lane, tl, .. + }) = self.definitions.get(name) + { + let (len, main_lane, count) = (*len, *main_lane, tl.element_count()); + if !self.spend(count as u64) { + return true; + } self.flush_voice(); let (at, lane) = (self.pos, self.current_voice_number); + let Some(end) = self.end_within_bounds(at, len) else { + return true; + }; let uid = self.current_uid(); let Some(VarDef::Music { tl, voices, .. }) = self.definitions.get(name) else { unreachable!() @@ -427,7 +732,7 @@ impl<'src> WalkState<'src> { for voice in voices { self.voice_part_map.insert(voice.clone(), uid); } - self.pos = at + len; + self.pos = end; self.voice_start = self.pos; return true; } @@ -436,6 +741,10 @@ impl<'src> WalkState<'src> { }; match def { VarDef::Parts(parts, voices) => { + let count: usize = parts.iter().map(|pb| pb.tl.element_count()).sum(); + if !self.spend(count as u64) { + return true; + } self.flush_voice(); let at = self.pos; let mut uids = Vec::new(); @@ -454,19 +763,7 @@ impl<'src> WalkState<'src> { } } VarDef::Music { .. } => unreachable!("handled above"), - VarDef::FiguredBass(entries) => { - let mut at = self.pos; - for entry in entries { - match entry { - FiguredBassEntry::Figure(fb) => { - let d = fb.duration.actual_duration(); - self.add_event_at(at, Event::FiguredBass(fb)); - at += d; - } - FiguredBassEntry::Skip(dur) => at += dur.actual_duration(), - } - } - } + VarDef::FiguredBass(entries) => self.place_figures(entries), } true } @@ -482,21 +779,26 @@ impl<'src> WalkState<'src> { if let Some(ref prev) = self.prev_pitch { // In relative mode: find closest pitch within a fourth, then apply marks let inferred_octave = find_relative_octave(prev, step); - let octave = inferred_octave + octave_marks; + let octave = self.bounded_octave(inferred_octave.saturating_add(octave_marks)); let p = Pitch::with_alter(step, alter, octave); self.prev_pitch = Some(p); p } else { // First note after \relative: use the reference pitch's octave let base_oct = self.relative_ref.as_ref().map(|r| r.octave).unwrap_or(4); - let octave = base_oct + octave_marks; + let octave = self.bounded_octave(base_oct.saturating_add(octave_marks)); let p = Pitch::with_alter(step, alter, octave); self.prev_pitch = Some(p); p } } else { - // Absolute mode: octave marks relative to LilyPond c (octave 3 in our numbering) - let octave = 3 + octave_marks; + // Absolute mode: octave marks relative to LilyPond c (octave 3 in + // our numbering), and `\fixed`'s octaves. + let octave = self.bounded_octave( + octave_marks + .saturating_add(3) + .saturating_add(self.fixed_octaves), + ); Pitch::with_alter(step, alter, octave) }; // Apply any active \transpose intervals @@ -505,10 +807,51 @@ impl<'src> WalkState<'src> { .transpose_stack .iter() .map(|(from, to)| to.midi_number() - from.midi_number()) - .sum(); - pitch.transposed(total_semitones) + .fold(0, i32::saturating_add); + let p = pitch.transposed(total_semitones.clamp(-1200, 1200)); + Pitch { + octave: self.bounded_octave(p.octave), + ..p + } } else { pitch } } + + /// `octave` if within [`PITCH_OCTAVES`], else the nearest octave in it + /// (and the reading stops): a relative passage cannot climb for ever, and + /// every emitter writes each octave mark out. + fn bounded_octave(&mut self, octave: i32) -> i32 { + if !PITCH_OCTAVES.contains(&octave) { + self.refuse(format!( + "a pitch lies in octave {octave}, outside {}..={}", + PITCH_OCTAVES.start(), + PITCH_OCTAVES.end() + )); + } + octave.clamp(*PITCH_OCTAVES.start(), *PITCH_OCTAVES.end()) + } +} + +/// Whether `block` contains a reference to the variable `name` (`\name`). +fn mentions(block: Node, source: &str, name: &str) -> bool { + let mut cursor = block.walk(); + loop { + let node = cursor.node(); + if node.kind() == "escaped_word" + && node + .utf8_text(source.as_bytes()) + .is_ok_and(|t| t.strip_prefix('\\') == Some(name)) + { + return true; + } + if cursor.goto_first_child() { + continue; + } + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() || cursor.node() == block { + return false; + } + } + } } diff --git a/src/adapters/ly_to_ir/syntax.rs b/src/adapters/ly_to_ir/syntax.rs new file mode 100644 index 0000000..3824894 --- /dev/null +++ b/src/adapters/ly_to_ir/syntax.rs @@ -0,0 +1,282 @@ +//! Syntax diagnostics from the tree alone: what tree-sitter could not parse, +//! and tokens no LilyPond construct accepts where they stand. + +use tree_sitter::{Node, Tree}; + +use crate::diagnostics::{Columns, Diagnostic, Severity}; +use crate::ir::language::{parse_pitch_name, PitchLanguage}; + +/// Every syntax error in `tree`, parsed from `source`. +pub(crate) fn syntax_diagnostics(tree: &Tree, source: &str) -> Vec { + let mut out = Vec::new(); + let cols = &mut Columns::default(); + let mut cursor = tree.walk(); + loop { + let node = cursor.node(); + let mut descend = true; + if node.is_error() { + out.push( + unclosed(cols, source, node) + .unwrap_or_else(|| unexpected(cols, source, node, node.end_byte())), + ); + descend = false; + } else if node.is_missing() { + out.push(Diagnostic::counted( + cols, + source, + node, + node.end_byte(), + Severity::Error, + "missing-token", + format!("missing `{}`", node.kind()), + )); + } else if node.kind() == "punctuation" && text(source, node) == ">" { + out.extend(stray_angle(cols, source, node)); + } else if node.kind() == "embedded_scheme" { + out.extend(after_scheme(cols, source, node)); + out.extend(markup_override(cols, source, node)); + } + if descend && cursor.goto_first_child() { + continue; + } + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() { + return out; + } + } + } +} + +fn text<'a>(source: &'a str, node: Node) -> &'a str { + source.get(node.byte_range()).unwrap_or("") +} + +/// `unexpected `…`` for the text from `node` to byte `end` (its first line, +/// shortened). +fn unexpected(cols: &mut Columns, source: &str, node: Node, end: usize) -> Diagnostic { + let snippet = source.get(node.start_byte()..end).unwrap_or(""); + let snippet = snippet.lines().next().unwrap_or("").trim(); + let short: String = snippet.chars().take(30).collect(); + let message = match (short.is_empty(), short.len() < snippet.len()) { + (true, _) => "syntax error".to_string(), + (false, true) => format!("unexpected `{short}…`"), + (false, false) => format!("unexpected `{short}`"), + }; + Diagnostic::counted( + cols, + source, + node, + end, + Severity::Error, + "syntax-error", + message, + ) +} + +/// An ERROR node holding a bracket or quote it never closes, as tree-sitter +/// makes of `{ c'4 d'4` at the end of the input: `missing `}``, at the opener. +fn unclosed(cols: &mut Columns, source: &str, error: Node) -> Option { + let mut open: Vec<(Node, &str)> = Vec::new(); + let mut cursor = error.walk(); + for child in error.children(&mut cursor) { + let closer = match child.kind() { + "{" => "}", + "<<" => ">>", + "<" => ">", + "\"" if open.last().is_some_and(|(_, c)| *c == "\"") => { + open.pop(); + continue; + } + "\"" => "\"", + kind => { + if open.last().is_some_and(|(_, c)| *c == kind) { + open.pop(); + } + continue; + } + }; + open.push((child, closer)); + } + let (opener, closer) = open.pop()?; + Some(Diagnostic::counted( + cols, + source, + opener, + opener.end_byte(), + Severity::Error, + "missing-token", + format!("missing `{closer}`: `{}` is never closed", opener.kind()), + )) +} + +/// A `>` outside a chord that is not an accent (`->`, `^>`, `_>`): what is +/// left of `<< … >>` without its `<<`, or of a chord without its `<`. The +/// grammar reads it as punctuation. +fn stray_angle(cols: &mut Columns, source: &str, node: Node) -> Option { + let is_angle = |n: Option, adjacent: usize| { + n.is_some_and(|n| { + n.kind() == "punctuation" && text(source, n) == ">" && { + let (a, b) = (n.start_byte(), n.end_byte()); + a == adjacent || b == adjacent + } + }) + }; + let prev = previous(node); + // The second `>` of a `>>`: reported with the first. + if is_angle(prev, node.start_byte()) { + return None; + } + if prev.is_some_and(|p| matches!(text(source, p), "-" | "^" | "_")) { + return None; + } + let next = node.next_sibling(); + let end = if is_angle(next, node.end_byte()) { + next.map_or(node.end_byte(), |n| n.end_byte()) + } else { + node.end_byte() + }; + let message = if end > node.end_byte() { + "`>>` without a matching `<<`" + } else { + "`>` outside a chord" + }; + Some(Diagnostic::counted( + cols, + source, + node, + end, + Severity::Error, + "syntax-error", + message.to_string(), + )) +} + +/// What is left of a Scheme expression that lost an opening `(`: `#(foo 1 . +/// 2)` reads as `#foo` and leaves `1 . 2)` to LilyPond. Outside markup, a +/// dotted pair's `.` never follows a Scheme value; nor a parenthesis, right +/// after it or after one more word that is no note, or a fraction +/// (`#string->symbol name)`): a slur follows a note. Unless the value is a +/// command's argument, and the parenthesis maybe the command's post-event +/// (`-\tweak color #red (`, `\vshape #'(…) (`). Nor, at the top level, a +/// number, string or quote on the line of a Scheme expression that starts it +/// (`#set-global-staff-size 20)`). +fn after_scheme(cols: &mut Columns, source: &str, node: Node) -> Option { + let mut next = node.next_sibling(); + while next.is_some_and(|n| n.kind() == "comment") { + next = next.and_then(|n| n.next_sibling()); + } + let next = next?; + let (kind, t) = (next.kind(), text(source, next)); + let top = node + .parent() + .is_some_and(|p| p.kind() == "lilypond_program"); + let line_start = node.start_byte() - node.start_position().column; + let starts_line = source + .get(line_start..node.start_byte()) + .is_some_and(|s| s.trim().is_empty()); + let same_line = next.start_position().row == node.end_position().row; + // Back over the other arguments to a command (not `\set`/`\override`, + // whose value follows `=`), or to a post-event's direction mark. + let post_event = || { + for p in std::iter::successors(node.prev_sibling(), |p| p.prev_sibling()) { + match (p.kind(), text(source, p)) { + ("punctuation", "-" | "^" | "_") | ("escaped_word", _) => return true, + ("embedded_scheme" | "symbol" | "string" | "comment", _) => {} + _ => return false, + } + } + false + }; + let is_paren = |n: Node| n.kind() == "punctuation" && matches!(text(source, n), "(" | ")"); + let no_note = |n: Node| { + let t = text(source, n); + n.kind() == "fraction" + || (n.kind() == "symbol" + && !matches!(t, "r" | "R" | "s" | "q") + && PitchLanguage::ALL + .iter() + .all(|&lang| parse_pitch_name(t, lang).is_none())) + }; + // `#foo word )`: the parenthesis is the one reported. + let (next, kind, t) = match next.next_sibling() { + Some(after) if no_note(next) && is_paren(after) && !post_event() => { + (after, after.kind(), text(source, after)) + } + _ => (next, kind, t), + }; + let stray = (kind == "punctuation" && t == ".") + || (kind == "punctuation" && matches!(t, "(" | ")") && !post_event()) + || (top + && starts_line + && same_line + && (matches!( + kind, + "unsigned_integer" | "decimal_number" | "fraction" | "string" + ) || (kind == "punctuation" && t == "'"))); + (stray && !in_markup(source, next)).then(|| { + Diagnostic::counted( + cols, + source, + next, + next.end_byte(), + Severity::Error, + "syntax-error", + format!("unexpected `{t}` after a Scheme expression"), + ) + }) +} + +/// `\override #'key` in markup: the command takes a pair (`#'(key . value)`), +/// and a quoted symbol or number never is one. +fn markup_override(cols: &mut Columns, source: &str, node: Node) -> Option { + let command = previous(node).filter(|p| text(source, *p) == r"\override")?; + let datum = text(source, node).strip_prefix('#')?; + let quoted = datum + .strip_prefix('\'') + .or_else(|| datum.strip_prefix('`'))?; + (!quoted.is_empty() && !quoted.starts_with('(')).then(|| { + Diagnostic::counted( + cols, + source, + node, + node.end_byte(), + Severity::Error, + "syntax-error", + format!( + "`{}` takes a pair, like `#'(key . value)`, not `{}`", + text(source, command), + text(source, node) + ), + ) + }) +} + +/// Whether `node` stands in markup, where any word or mark is text: it, or a +/// block around it, follows `\markup` through markup commands and their +/// Scheme arguments only (`\markup \hspace #1 .`, `\markup { \line { … } }`). +fn in_markup(source: &str, node: Node) -> bool { + let mut at = Some(node); + while let Some(n) = at { + let mut prev = n.prev_sibling(); + while let Some(p) = prev { + match p.kind() { + "escaped_word" if matches!(text(source, p), "\\markup" | "\\markuplist") => { + return true; + } + "escaped_word" | "embedded_scheme" | "comment" => prev = p.prev_sibling(), + _ => break, + } + } + at = n.parent(); + } + false +} + +/// The sibling before `node`, skipping comments. +fn previous(node: Node) -> Option { + let mut prev = node.prev_sibling(); + while prev.is_some_and(|p| p.kind() == "comment") { + prev = prev.and_then(|p| p.prev_sibling()); + } + prev +} diff --git a/src/adapters/ly_to_ir/tests.rs b/src/adapters/ly_to_ir/tests.rs index 7877721..7c99773 100644 --- a/src/adapters/ly_to_ir/tests.rs +++ b/src/adapters/ly_to_ir/tests.rs @@ -2224,6 +2224,46 @@ melB = { g'4 a' b' c'' } assert_eq!(shifts[1].size, 0); } + #[test] + fn test_ottava_sizes_are_musicxml_intervals() { + // `\ottava #2` is a 15ma (MusicXML size 15, not 16); past three + // octaves the value is clamped (`#2147483647` overflowed `n * 8`). + let adapter = LyToIrAdapter::new(); + for (n, kind, size) in [ + ("2", "up", 15), + ("-2", "down", 15), + ("3", "up", 22), + ("2147483647", "up", 22), + ("-2147483648", "down", 22), + ] { + let score = adapter + .convert_str(&format!("{{ \\ottava #{n} c''4 }}")) + .unwrap(); + let shift = score.parts()[0].measures[0] + .directions + .iter() + .find_map(|d| d.octave_shift.clone()) + .unwrap(); + assert_eq!( + (shift.shift_type.as_str(), shift.size), + (kind, size), + "#{n}" + ); + } + // Written back, the octaves come out the same. + for n in ["2", "-2", "-1"] { + let score = adapter + .convert_str(&format!("{{ \\ottava #{n} c''4 }}")) + .unwrap(); + let ly = crate::adapters::FromIrAdapter::convert( + &crate::adapters::ir_to_ly::IrToLyAdapter::new(), + &score, + ) + .unwrap(); + assert!(ly.contains(&format!("\\ottava #{n}")), "{n}: {ly}"); + } + } + #[test] fn test_layout_break_parsing() { let adapter = LyToIrAdapter::new(); @@ -3393,6 +3433,33 @@ lower = \relative c { \partial 8 c8 | d8 e f g a b | c8 b a g f e } ); } + /// Inside `\relative`, LilyPond makes the written body relative once and + /// copies it: the copies of an unfold have the same pitches (lytk used to + /// read each pass relative to the previous one, climbing an octave step + /// per pass; LilyPond's page-layout tests reached octave 128 that way). + #[test] + fn test_repeat_unfold_in_relative_repeats_the_same_pitches() { + let score = LyToIrAdapter::new() + .convert_str(r#"\relative { \repeat unfold 10 { \repeat unfold 4 { c''4 } } d4 }"#) + .unwrap(); + let octaves: Vec = score.parts()[0] + .measures + .iter() + .flat_map(|m| &m.voices) + .flat_map(|v| &v.elements) + .filter_map(|e| match e { + VoiceElement::Note(n) => Some(n.pitch.octave), + _ => None, + }) + .collect(); + assert_eq!(octaves.len(), 41); + assert!( + octaves[..40].iter().all(|&o| o == octaves[0]), + "{octaves:?}" + ); + assert_eq!(octaves[40], octaves[0], "d4 is relative to the last c"); + } + /// `\repeat unfold N { … }` writes the body out N times (was emitted once, /// dropping N−1 copies). #[test] @@ -3518,3 +3585,9 @@ mod transposition { } } } + +#[test] +fn builtins_are_sorted_for_binary_search() { + let names = super::builtins::BUILTINS; + assert!(names.windows(2).all(|w| w[0] < w[1])); +} diff --git a/src/adapters/ly_to_ir/text.rs b/src/adapters/ly_to_ir/text.rs new file mode 100644 index 0000000..2ea334a --- /dev/null +++ b/src/adapters/ly_to_ir/text.rs @@ -0,0 +1,266 @@ +//! Text in LilyPond, decoded as LilyPond reads it: strings, Scheme strings, +//! markup, and the fields of `\header` blocks. + +use tree_sitter::Node; + +fn slice<'a>(source: &'a str, node: Node) -> &'a str { + source.get(node.byte_range()).unwrap_or("") +} + +/// A string node's text, decoded as LilyPond's lexer does +/// (`Lily_lexer::escaped_char`): `\n`, `\t`, `\\`, `\"` and `\'`; any other +/// backslash stays, with the character after it. +pub(crate) fn string_value(source: &str, node: Node) -> String { + let mut out = String::new(); + let mut cursor = node.walk(); + for child in node.children(&mut cursor) { + let t = slice(source, child); + match child.kind() { + "string_fragment" => out.push_str(t), + "escape_sequence" => match t { + "\\n" => out.push('\n'), + "\\t" => out.push('\t'), + "\\\\" => out.push('\\'), + "\\\"" => out.push('"'), + "\\'" => out.push('\''), + other => out.push_str(other), + }, + _ => {} + } + } + out +} + +/// The text of a Scheme string literal (`#"…"`), with its common escapes +/// decoded (`\\`, `\"`, `\n`, `\t`, `\r`; others stay); `None` for any other +/// Scheme value. +pub(crate) fn scheme_string(text: &str) -> Option { + let inner = text.trim().strip_prefix('#').unwrap_or(text).trim(); + let inner = inner.strip_prefix('"')?.strip_suffix('"')?; + let mut out = String::with_capacity(inner.len()); + let mut chars = inner.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some(e @ ('\\' | '"')) => out.push(e), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } + Some(out) +} + +/// The index past a markup expression starting at `children[i]` (just after +/// `\markup`): markup commands with their Scheme arguments, then one block, +/// string or word. It stops at a command that starts a new top-level +/// construct. +pub(crate) fn markup_end(source: &str, children: &[Node], mut i: usize) -> usize { + while let Some(node) = children.get(i) { + match node.kind() { + "escaped_word" + if !matches!( + slice(source, *node), + "\\score" + | "\\book" + | "\\bookpart" + | "\\header" + | "\\paper" + | "\\layout" + | "\\midi" + | "\\version" + | "\\include" + | "\\language" + | "\\markup" + | "\\markuplist" + | "\\new" + | "\\context" + | "\\relative" + | "\\transpose" + ) => + { + i += 1 + } + "embedded_scheme" => i += 1, + "expression_block" | "string" | "symbol" => return i + 1, + _ => return i, + } + } + i +} + +/// The plain text of markup: its strings (decoded) and words, with one space +/// where the source has anything between them; commands and their Scheme +/// arguments are left out. `\markup { \bold "J. S." Bach }` is `J. S. Bach`. +pub(crate) fn markup_text(source: &str, nodes: &[Node]) -> String { + let mut out = String::new(); + let mut last_end = None; + let mut piece = |out: &mut String, text: &str, start: usize, end: usize| { + if text.is_empty() { + return; + } + let gap = last_end.is_some_and(|e| e < start); + if gap && !out.is_empty() && !out.ends_with(' ') { + out.push(' '); + } + out.push_str(text); + last_end = Some(end); + }; + for &root in nodes { + let mut cursor = root.walk(); + 'walk: loop { + let n = cursor.node(); + let descend = match n.kind() { + "string" => { + piece( + &mut out, + &string_value(source, n), + n.start_byte(), + n.end_byte(), + ); + false + } + "escaped_word" | "embedded_scheme" | "comment" => false, + "{" | "}" => false, + _ if n.child_count() == 0 => { + piece(&mut out, slice(source, n), n.start_byte(), n.end_byte()); + false + } + _ => true, + }; + if descend && cursor.goto_first_child() { + continue; + } + loop { + if cursor.node() == root { + break 'walk; + } + if cursor.goto_next_sibling() { + continue 'walk; + } + if !cursor.goto_parent() { + break 'walk; + } + } + } + } + out.trim().to_string() +} + +/// A field of a `\header` block: its key, its value as text, the byte range +/// of `key = value`, and the `\score` block it belongs to (by the order of +/// `\score` blocks in the file), if any. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct HeaderField { + pub key: String, + pub value: String, + pub start: usize, + pub end: usize, + pub score: Option, +} + +/// The fields of a `\header { … }` block, in order: a string value decoded, +/// a `\markup` value as its plain text, `#"…"` as its string. Fields with +/// other values (`##f`, Scheme) are left out. +pub(crate) fn header_block_fields(source: &str, block: Node) -> Vec { + let mut cursor = block.walk(); + let children: Vec = block + .children(&mut cursor) + .filter(|n| n.kind() != "comment") + .collect(); + let mut out = Vec::new(); + let mut i = 0; + while i < children.len() { + let n = children[i]; + let is_eq = |k: usize| { + children + .get(k) + .is_some_and(|e| e.kind() == "punctuation" && slice(source, *e) == "=") + }; + let key = match n.kind() { + "assignment_lhs" if is_eq(i + 1) => slice(source, n).trim().to_string(), + // A quoted key: `"key" = …`. + "string" if is_eq(i + 1) => string_value(source, n), + _ => { + i += 1; + continue; + } + }; + let j = i + 2; + let Some(&v) = children.get(j) else { + break; + }; + let (value, end) = match v.kind() { + "string" => (Some(string_value(source, v)), j + 1), + "embedded_scheme" => (scheme_string(slice(source, v)), j + 1), + "escaped_word" if matches!(slice(source, v), "\\markup" | "\\markuplist") => { + let end = markup_end(source, &children, j + 1); + (Some(markup_text(source, &children[j + 1..end])), end) + } + _ => (None, j + 1), + }; + if let Some(value) = value { + out.push(HeaderField { + key, + value, + start: n.start_byte(), + end: children[end - 1].end_byte(), + score: None, + }); + } + i = end; + } + out +} + +/// Every `\header` field of a syntax tree, in source order, each with the +/// `\score` block it is in. +pub(crate) fn header_fields_of(source: &str, root: Node) -> Vec { + let mut scores: Vec<(usize, usize)> = Vec::new(); + let mut out = Vec::new(); + let mut cursor = root.walk(); + 'walk: loop { + let n = cursor.node(); + if n.kind() == "escaped_word" { + let mut next = n.next_sibling(); + while next.is_some_and(|x| x.kind() == "comment") { + next = next.and_then(|x| x.next_sibling()); + } + if let Some(block) = next.filter(|b| b.kind() == "expression_block") { + match slice(source, n) { + "\\score" => scores.push((block.start_byte(), block.end_byte())), + "\\header" => { + let at = n.start_byte(); + let score = scores.iter().rposition(|&(s, e)| s <= at && at < e); + out.extend( + header_block_fields(source, block) + .into_iter() + .map(|f| HeaderField { score, ..f }), + ); + } + _ => {} + } + } + } + if cursor.goto_first_child() { + continue; + } + loop { + if cursor.goto_next_sibling() { + continue 'walk; + } + if !cursor.goto_parent() { + break 'walk; + } + } + } + out +} diff --git a/src/adapters/ly_to_ir/walk.rs b/src/adapters/ly_to_ir/walk.rs index a3c8661..dfacc55 100644 --- a/src/adapters/ly_to_ir/walk.rs +++ b/src/adapters/ly_to_ir/walk.rs @@ -1,13 +1,13 @@ use tree_sitter::Node; use crate::ir::duration::Frac; -use crate::ir::language::{parse_pitch_name, PitchLanguage, PitchMode}; +use crate::ir::language::{parse_pitch_name, PitchMode}; use crate::ir::pitch::Pitch; use super::chord_mode::parse_chordmode_block; use super::consume::{ block_contains_named_context, extract_string_value, parse_paper_block, parse_with_block, - score_block_output_types, + score_block_output_types, skip_markup, }; use super::figured_bass::parse_figuremode_block; use super::lyrics::{extract_lyricsto_voice, parse_lyric_block}; @@ -15,12 +15,22 @@ use super::merge::{assign_piano_direction_staff, part_is_dynamics_only}; use super::modifiers::{consume_relative, consume_transpose}; use super::music::walk_music_block; use super::state::{PartBuild, VarDef, WalkState}; +use super::text::header_block_fields; +use crate::diagnostics::{Diagnostic, Severity}; use crate::ir::timeline::{Event, Timeline}; -/// Walk the `lilypond_program` root node. -pub(super) fn walk_program(state: &mut WalkState, root: Node) { - let mut cursor = root.walk(); - let children: Vec = root.children(&mut cursor).collect(); +/// Walk the `lilypond_program` root node. Each top-level music expression +/// and each `\score` block is a movement, in order, as LilyPond makes a score +/// of each. +pub(super) fn walk_program<'a>(state: &mut WalkState<'a>, root: Node<'a>) { + walk_top(state, root); + close_movement(state); +} + +/// The top level, or the inside of a `\book` or `\bookpart` block. +fn walk_top<'a>(state: &mut WalkState<'a>, parent: Node<'a>) { + let mut cursor = parent.walk(); + let children: Vec = parent.children(&mut cursor).collect(); let mut i = 0; while i < children.len() { @@ -35,20 +45,20 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { } "\\language" => { // \language "english" - if let Some(next) = children.get(i + 1) { - if next.kind() == "string" { - let lang_str = extract_string_value(state, *next); - state.language = PitchLanguage::from_str_loose(&lang_str) - .unwrap_or(PitchLanguage::Nederlands); - i += 1; // skip string - } + if let Some(&next) = children.get(i + 1).filter(|n| n.kind() == "string") { + state.set_language(next); + i += 1; // skip string } } "\\header" => { - // \header { ... } + // The book's \header: every movement's, unless it + // sets the field itself. if let Some(next) = children.get(i + 1) { if next.kind() == "expression_block" { - walk_header(state, *next); + let fields = header_block_fields(state.source, *next); + state + .book_header + .extend(fields.into_iter().map(|f| (f.key, f.value))); i += 1; } } @@ -60,45 +70,31 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { // Skip MIDI-only score blocks (have \midi but no \layout) let (has_layout, has_midi) = score_block_output_types(state, *next); if has_midi && !has_layout { + state.warn( + node, + "skipped-score", + "a `\\score` with `\\midi` and no `\\layout` prints nothing: \ + not read" + .to_string(), + ); i += 1; // skip the expression_block i += 1; continue; } - // Each \score block is its own movement. - state.flush_voice(); - let saved_parts = std::mem::take(&mut state.parts); - let saved_counter = state.part_counter; - let saved_pending_lyrics = - std::mem::take(&mut state.pending_lyrics); - let saved_added_lyrics = std::mem::take(&mut state.added_lyrics); - let saved_pending_harmonies = - std::mem::take(&mut state.pending_harmonies); - let saved_voice_map = std::mem::take(&mut state.voice_part_map); - state.part_counter = 0; - state.set_pos(Frac::from_integer(0)); - // `\partial` is per-movement: reset so a pickup in - // one \score block doesn't leak into the next. - state.metadata.partial_duration = None; - + // Each \score block is its own movement, with + // its own \header. + close_movement(state); + let saved_metadata = std::mem::take(&mut state.metadata); walk_score_block(state, *next); - if let Some(score) = super::assemble_score(state) { - state.completed_scores.push(score); - } - - // Restore saved state - state.parts = saved_parts; - state.part_counter = saved_counter; - state.pending_lyrics = saved_pending_lyrics; - state.added_lyrics = saved_added_lyrics; - state.pending_harmonies = saved_pending_harmonies; - state.voice_part_map = saved_voice_map; - + finish_movement(state, node); + state.metadata = saved_metadata; i += 1; } } } "\\relative" => { // Top-level \relative c' { ... } + begin_movement(state, node); state.in_relative = true; state.mode = PitchMode::Relative; i += 1; @@ -106,6 +102,7 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { continue; } "\\transpose" => { + begin_movement(state, node); i += 1; i = consume_transpose(state, &children, i); continue; @@ -132,22 +129,57 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { } } } + "\\layout" | "\\midi" => { + // Output settings and context definitions: not music. + if children + .get(i + 1) + .is_some_and(|n| n.kind() == "expression_block") + { + i += 1; + } + } + "\\markup" | "\\markuplist" => { + // Top-level text: not music. + i = skip_markup(state, &children, i + 1); + continue; + } + "\\book" | "\\bookpart" => { + // A book's scores and top-level music are movements + // too, and its \header is the book's. + if let Some(&next) = children.get(i + 1) { + if next.kind() == "expression_block" { + walk_top(state, next); + i += 1; + } + } + } _ => { - // Top-level escaped words we don't handle + // A music variable at the top level is a music + // expression (a movement); other commands aren't read. + let name = text.trim_start_matches('\\'); + if state.definitions.contains_key(name) { + begin_movement(state, node); + state.resolve_variable(name); + } else { + state.unread_command(node, name); + } } } } "expression_block" => { - // Bare { ... } at top level: treat as a single anonymous part + // Bare { ... } at top level: a movement of its own + begin_movement(state, node); walk_music_block(state, node); } "parallel_music" => { // Top-level `<< ... >>` (e.g. emitted multi-staff music with no - // explicit \score wrapper) is an implicit score. Walk it directly. + // explicit \score wrapper) is a movement. Walk it directly. + begin_movement(state, node); walk_parallel_music(state, node); } "named_context" => { - // Top-level `\new Staff { ... }` etc. — an implicit score. + // Top-level `\new Staff { ... }` etc. — a movement. + begin_movement(state, node); let (context, name) = extract_named_context(state, node); i += 1; i = walk_context_body(state, &children, i, &context, &name); @@ -166,6 +198,7 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { result }; if !var_name.is_empty() { + state.assigned.insert(var_name.clone()); // Skip the "=" punctuation, then capture the body if let Some(eq) = children.get(i + 1) { if eq.kind() == "punctuation" && state.text(*eq) == "=" { @@ -204,6 +237,22 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { j += 1; continue; } + // Read from the block, which knows its mode. + if ew == "\\drummode" || ew == "\\drums" { + j += 1; + continue; + } + if ew == "\\fixed" { + // Its pitch and marks; the block reads them. + j += 1; + while children.get(j).is_some_and(|n| { + n.kind() == "symbol" + || matches!(state.text(*n), "'" | ",") + }) { + j += 1; + } + continue; + } if ew == "\\relative" { state.in_relative = true; state.mode = PitchMode::Relative; @@ -288,6 +337,7 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { if next.kind() == "expression_block" && is_figuremode { // Parse figuremode block into figured bass entries let fb_measures = parse_figuremode_block(state, *next); + state.spend(fb_measures.len() as u64); state .definitions .insert(var_name, VarDef::FiguredBass(fb_measures)); @@ -307,16 +357,22 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { // A `\relative` definition keeps its own // reference; a plain block is read again // where it's used inside `\relative`. - let range = (!is_relative) - .then(|| (block.start_byte(), block.end_byte())); + let range = (!is_relative).then(|| { + (block.start_byte(), block.end_byte(), state.language) + }); let depth = u32::from(is_relative); + // A chord-mode block, read above as + // harmonies, is read as notes too. + let quiet = u32::from(is_chordmode); let (def, ()) = capture_variable( state, has_named_context, range, |state| { state.relative_depth += depth; + state.quiet += quiet; walk_music_block(state, block); + state.quiet -= quiet; state.relative_depth -= depth; }, ); @@ -347,59 +403,185 @@ pub(super) fn walk_program(state: &mut WalkState, root: Node) { } } } + "string" if children.get(i + 1).is_some_and(|n| state.text(*n) == "=") => { + // `"name" = …`, a quoted identifier: `"\\|" = …` defines `\|`. + let raw = state.text(node); + let name = raw + .get(1..raw.len().saturating_sub(1)) + .unwrap_or("") + .replace("\\\\", "\\"); + let name = name.strip_prefix('\\').unwrap_or(&name).to_string(); + state.assigned.insert(name); + } + "embedded_scheme" => { + // `#(define (name …) …)` and its kin define `\name` too. + let names: Vec = scheme_definitions(state.text(node)) + .map(str::to_string) + .collect(); + state.assigned.extend(names); + } + "symbol" => top_level_word(state, &children, i), _ => {} } i += 1; } } -/// Walk a `\header { ... }` block. -pub(super) fn walk_header(state: &mut WalkState, block: Node) { - let mut cursor = block.walk(); - let children: Vec = block.children(&mut cursor).collect(); - let mut i = 0; +/// The names a Scheme expression defines: `(define name …)`, `(define (name +/// …) …)`, `(define-public …)`, `(define-markup-command (name …) …)`. +fn scheme_definitions(text: &str) -> impl Iterator { + text.match_indices("(define").filter_map(|(at, _)| { + // Past the rest of the keyword (`-public`, `-markup-command`, …). + let rest = text[at + "(define".len()..] + .split_once(char::is_whitespace)? + .1; + let rest = rest.trim_start(); + let rest = rest.strip_prefix('(').unwrap_or(rest); + let end = rest + .find(|c: char| c.is_whitespace() || c == '(' || c == ')') + .unwrap_or(rest.len()); + (end > 0).then(|| &rest[..end]) + }) +} - while i < children.len() { - let node = children[i]; - if node.kind() == "assignment_lhs" { - let key = { - // assignment_lhs has a child symbol - let mut c = node.walk(); - let result = node - .children(&mut c) - .find(|n| n.kind() == "symbol") - .map(|n| state.text(n).to_string()) - .unwrap_or_default(); - result - }; +/// Start a movement for the top-level music expression at `node`, closing +/// the one before it (`\addlyrics` after an expression still belongs to it). +fn begin_movement<'a>(state: &mut WalkState<'a>, node: Node<'a>) { + close_movement(state); + state.open_movement = Some(node); +} - // Skip the "=" punctuation - // Then find the next string value - let mut j = i + 1; - while j < children.len() { - let val_node = children[j]; - if val_node.kind() == "string" { - let val = extract_string_value(state, val_node); - match key.as_str() { - "title" => state.metadata.title = Some(val), - "subtitle" => state.metadata.subtitle = Some(val), - "composer" => state.metadata.composer = Some(val), - "arranger" => state.metadata.arranger = Some(val), - "poet" | "lyricist" => state.metadata.lyricist = Some(val), - k if !k.is_empty() => { - state.metadata.extra.insert(k.to_string(), val); - } - _ => {} - } - i = j; - break; - } else if val_node.kind() == "assignment_lhs" || val_node.kind() == "}" { - break; - } - j += 1; +/// Close the top-level music expression being read, if any, into a movement. +fn close_movement(state: &mut WalkState) { + if let Some(anchor) = state.open_movement.take() { + finish_movement(state, anchor); + } +} + +/// Assemble what was walked into a movement, and start afresh: each +/// movement has its own parts, positions and lyrics. +fn finish_movement(state: &mut WalkState, anchor: Node) { + if let Some(score) = super::assemble_score(state) { + state.completed_scores.push(score); + let n = state.completed_scores.len(); + if n > 1 { + state.later_movements.push(Diagnostic::at( + state.source, + anchor, + Severity::Warning, + "dropped-music", + format!("movement {n} is not read: this reading takes the first movement only"), + )); + } + } + state.parts.clear(); + state.current_voice.clear(); + state.part_counter = 0; + state.pending_lyrics.clear(); + state.added_lyrics.clear(); + state.pending_harmonies.clear(); + state.voice_part_map.clear(); + let zero = Frac::from_integer(0); + (state.pos, state.voice_start, state.origin) = (zero, zero, zero); + state.metadata.partial_duration = None; +} + +/// A top-level word starting the file, or following `\version "…"` (or +/// `\include`, `\language`): text that is not LilyPond, or notes outside +/// braces, which LilyPond rejects too. Elsewhere a word may be a command's +/// argument or an assignment's value: not judged. +fn top_level_word(state: &mut WalkState, children: &[Node], i: usize) { + let significant = |j: usize| (0..j).rev().find(|&k| children[k].kind() != "comment"); + let at_start = match significant(i) { + None => true, + Some(p) => { + children[p].kind() == "string" + && significant(p).is_some_and(|q| { + matches!( + state.text(children[q]), + "\\version" | "\\include" | "\\language" + ) + }) + } + }; + if !at_start { + return; + } + // The run of words, numbers and marks from here (`violin.1 = …` is an + // assignment). + let run: Vec = children[i..] + .iter() + .copied() + .filter(|n| n.kind() != "comment") + .take_while(|n| { + matches!( + n.kind(), + "symbol" | "punctuation" | "unsigned_integer" | "decimal_number" + ) && state.text(*n) != "=" + }) + .collect(); + let assigns = children[i..] + .iter() + .filter(|n| n.kind() != "comment") + .nth(run.len()) + .is_some_and(|n| state.text(*n) == "="); + if assigns { + return; + } + let is_note = |text: &str| { + parse_pitch_name(text, state.language).is_some() || matches!(text, "r" | "R" | "s" | "q") + }; + let word = run + .iter() + .find(|n| n.kind() == "symbol" && !is_note(state.text(**n))); + match word { + Some(&w) => { + let text = state.text(w); + state.error( + w, + "not-lilypond", + format!("`{text}` is not LilyPond: a bare word at the top level"), + ); + } + // LilyPond: "syntax error, unexpected NOTENAME_PITCH". + None => state.error( + children[i], + "syntax-error", + "notes outside braces: music at the top level goes in `{ }`".to_string(), + ), + } +} + +/// Walk a `\score`'s `\header { ... }` block into the movement's metadata. +pub(super) fn walk_header(state: &mut WalkState, block: Node) { + for field in header_block_fields(state.source, block) { + set_header_field(&mut state.metadata, &field.key, field.value, true); + } +} + +/// Set a header field in `meta`: the fields the IR names (`poet` is the +/// lyricist) or one of `extra`. Unless `overwrite`, a field already set stays. +pub(super) fn set_header_field( + meta: &mut crate::ir::score::ScoreMetadata, + key: &str, + value: String, + overwrite: bool, +) { + let slot = match key { + "title" => &mut meta.title, + "subtitle" => &mut meta.subtitle, + "composer" => &mut meta.composer, + "arranger" => &mut meta.arranger, + "poet" | "lyricist" => &mut meta.lyricist, + _ => { + if overwrite || !meta.extra.contains_key(key) { + meta.extra.insert(key.to_string(), value); } + return; } - i += 1; + }; + if overwrite || slot.is_none() { + *slot = Some(value); } } @@ -444,6 +626,12 @@ pub(super) fn walk_score_block(state: &mut WalkState, block: Node) { } } } + "\\language" => { + if let Some(&next) = children.get(i + 1).filter(|n| n.kind() == "string") { + state.set_language(next); + i += 1; + } + } "\\addlyrics" => { // `MUSIC \addlyrics { ... }` — lyrics attach to the music // expression that immediately precedes. Flush the pending @@ -501,7 +689,9 @@ pub(super) fn walk_score_block(state: &mut WalkState, block: Node) { } _ => { let var_name = text.trim_start_matches('\\'); - state.resolve_variable(var_name); + if !state.resolve_variable(var_name) { + state.unread_command(node, var_name); + } } } } @@ -582,7 +772,7 @@ fn walk_parallel_music_voices(state: &mut WalkState, children: &[Node]) { state.relative_ref = saved_relative_ref; state.in_relative = saved_in_relative; state.stem_direction = saved_stem_direction.clone(); - state.current_voice_number = (voice_idx + 1) as u8; + state.current_voice_number = u8::try_from(voice_idx + 1).unwrap_or(u8::MAX); state.last_duration = saved_last_duration.clone(); state.tuplet_stack = saved_tuplet_stack.clone(); state.auto_beam_off = saved_auto_beam_off; @@ -671,7 +861,9 @@ fn walk_voice_branch(state: &mut WalkState, children: &[Node], branch_indices: & continue; } else { let var_name = text.trim_start_matches('\\'); - state.resolve_variable(var_name); + if !state.resolve_variable(var_name) { + state.unread_command(child, var_name); + } } } _ => {} @@ -828,7 +1020,10 @@ fn walk_parallel_music_staves(state: &mut WalkState, children: &[Node]) { } else { // A plain music variable is a simultaneous branch. begin_branch(state); - state.resolve_variable(text.trim_start_matches('\\')); + let var_name = text.trim_start_matches('\\'); + if !state.resolve_variable(var_name) { + state.unread_command(child, var_name); + } state.flush_voice(); end = end.max(state.pos); } @@ -906,7 +1101,7 @@ pub(super) fn walk_context_body( // Voice context: no new part — the voice's music goes into the current // staff (its own lane when it overlaps music already there). - if context == "Voice" { + if context == "Voice" || context == "DrumVoice" { i = skip_with_block(state, children, i); i = walk_body(state, children, i); if !name.is_empty() { @@ -1069,6 +1264,9 @@ pub(super) fn walk_context_body( .is_some_and(|pb| pb.context == context && (name.is_empty() || pb.part.name == name)); if !reenter { state.new_part(context, name); + if context == "DrumStaff" { + super::music::add_percussion_clef(state); + } } while i < children.len() { @@ -1111,25 +1309,15 @@ pub(super) fn walk_context_body( } } _ => { - let var_name = text.trim_start_matches('\\'); - if state.resolve_variable(var_name) { - i += 1; - } + i = walk_body(state, children, i); break; } } } - "expression_block" => { - walk_music_block(state, node); - i += 1; + _ => { + i = walk_body(state, children, i); break; } - "parallel_music" => { - walk_parallel_music(state, node); - i += 1; - break; - } - _ => break, } i += 1; } @@ -1239,7 +1427,14 @@ fn merge_piano_staff_parts( return; } - let num_staves = staves.len() as u8; + // Staff numbers are `u8`: a group of more staves cannot be one part. + let Ok(num_staves) = u8::try_from(staves.len()) else { + state.refuse(format!( + "a staff group has {} staves; at most 255 fit in one part", + staves.len() + )); + return; + }; let mut staves = staves.into_iter(); let first = staves.next().unwrap(); let mut base = PartBuild { @@ -1250,7 +1445,7 @@ fn merge_piano_staff_parts( for (k, mut pb) in std::iter::once(first).chain(staves).enumerate() { pb.tl.fold_spacer_lanes(); let top_lane = pb.tl.lanes.keys().copied().max().unwrap_or(0); - base.tl.absorb(pb.tl.into_staff(k as u8 + 1, lane_offset)); + base.tl.absorb(pb.tl.into_staff(k as u8 + 1, lane_offset)); // k < num_staves ≤ 255 lane_offset = lane_offset.saturating_add(top_lane); if pb.uid != base.uid { state.part_alias.insert(pb.uid, base.uid); @@ -1323,6 +1518,11 @@ fn walk_body(state: &mut WalkState, children: &[Node], i: usize) -> usize { walk_parallel_music(state, node); i + 1 } + // `\new Staff \new Voice { … }` + "named_context" => { + let (context, name) = extract_named_context(state, node); + walk_context_body(state, children, i + 1, &context, &name) + } "escaped_word" => { let text = state.text(node).to_string(); match text.as_str() { @@ -1332,6 +1532,20 @@ fn walk_body(state: &mut WalkState, children: &[Node], i: usize) -> usize { consume_relative(state, children, i + 1) } "\\transpose" => consume_transpose(state, children, i + 1), + // The block after a mode reads the mode itself. + "\\drummode" | "\\chordmode" | "\\figuremode" | "\\lyricmode" | "\\notemode" + | "\\drums" | "\\chords" | "\\figures" | "\\lyrics" => { + walk_body(state, children, i + 1) + } + "\\fixed" => { + let mut j = i + 1; + while children.get(j).is_some_and(|n| { + n.kind() == "symbol" || matches!(state.text(*n), "'" | ",") + }) { + j += 1; + } + walk_body(state, children, j) + } _ if state.resolve_variable(text.trim_start_matches('\\')) => i + 1, _ => i, } @@ -1345,13 +1559,15 @@ fn walk_body(state: &mut WalkState, children: &[Node], i: usize) -> usize { fn capture_variable( state: &mut WalkState, as_parts: bool, - block: Option<(usize, usize)>, + block: Option<(usize, usize, crate::ir::language::PitchLanguage)>, walk: impl FnOnce(&mut WalkState) -> R, ) -> (VarDef, R) { state.flush_voice(); let saved_parts = std::mem::take(&mut state.parts); let saved_voices = std::mem::take(&mut state.voice_part_map); let saved_pos = (state.pos, state.voice_start, state.origin); + // Parts a variable defines are numbered where it is used. + let saved_counter = state.part_counter; let (old_relative, old_mode, old_ref, old_prev) = ( state.in_relative, state.mode, @@ -1370,6 +1586,7 @@ fn capture_variable( let parts = std::mem::replace(&mut state.parts, saved_parts); let voices = std::mem::replace(&mut state.voice_part_map, saved_voices); (state.pos, state.voice_start, state.origin) = saved_pos; + state.part_counter = saved_counter; // Restore state so variable definitions don't leak context state.in_relative = old_relative; state.mode = old_mode; diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 5f25030..eb729c3 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -55,6 +55,53 @@ pub enum AdapterError { /// Adapter result type. pub type Result = std::result::Result; +/// Longest music the MusicXML, ABC and Humdrum readers accept, in whole notes +/// (the LilyPond and MIDI readers have the same bound): past it the bars cut +/// downstream would be millions. +pub(crate) const MAX_WHOLE_NOTES: u32 = 100_000; + +fn too_long() -> AdapterError { + AdapterError::Unsupported(format!( + "the music is longer than {MAX_WHOLE_NOTES} whole notes; lytk refuses input this large" + )) +} + +/// Refuse music longer than [`MAX_WHOLE_NOTES`]. +pub(crate) fn check_music_length(music: &crate::ir::music::Music) -> Result<()> { + if music.approx_length() > f64::from(MAX_WHOLE_NOTES) { + return Err(too_long()); + } + Ok(()) +} + +/// Refuse a score longer than [`MAX_WHOLE_NOTES`]: its longest part, each +/// measure as long as its longest voice. +pub(crate) fn check_score_length(score: &Score) -> Result<()> { + let seconds = |e: &crate::ir::note::VoiceElement| { + let d = e.metric_duration(); + *d.numer() as f64 / *d.denom() as f64 + }; + let longest = score + .parts() + .iter() + .map(|p| { + p.measures + .iter() + .map(|m| { + m.voices + .iter() + .map(|v| v.elements.iter().map(seconds).sum::()) + .fold(0.0, f64::max) + }) + .sum::() + }) + .fold(0.0, f64::max); + if longest > f64::from(MAX_WHOLE_NOTES) { + return Err(too_long()); + } + Ok(()) +} + // --------------------------------------------------------------------------- // Traits // --------------------------------------------------------------------------- diff --git a/src/adapters/mxml_to_ir/mod.rs b/src/adapters/mxml_to_ir/mod.rs index 5362da9..588c845 100644 --- a/src/adapters/mxml_to_ir/mod.rs +++ b/src/adapters/mxml_to_ir/mod.rs @@ -163,10 +163,12 @@ fn read_partwise_bytes(bytes: Vec) -> Result { }; // The firewall covers conversion too: panics there would otherwise escape // to the PyO3 boundary as aborts instead of AdapterError::Parse. - catch_read(|| { + let score = catch_read(|| { let mxml_score = musicxml::read_score_data_partwise(xml).map_err(AdapterError::Parse)?; convert_mxml_score(&mxml_score) - }) + })?; + super::check_score_length(&score)?; + Ok(score) } impl ToIrAdapter for MxmlToIrAdapter { diff --git a/src/diagnostics.rs b/src/diagnostics.rs new file mode 100644 index 0000000..87b3baf --- /dev/null +++ b/src/diagnostics.rs @@ -0,0 +1,160 @@ +//! Diagnostics: what a reader found wrong with its input, or could not read. +//! +//! The LilyPond reader reports syntax errors from the tree, and what its walk +//! drops or cannot represent. An error is input LilyPond itself rejects; a +//! warning is input lytk does not read (an unknown command, an include it does +//! not follow) or cannot represent, and reads around. + +use std::fmt; + +use tree_sitter::Node; + +/// How serious a [`Diagnostic`] is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Severity { + /// LilyPond rejects the input. + Error, + /// lytk does not read, or cannot represent, this part of the input. + Warning, +} + +impl Severity { + pub fn as_str(self) -> &'static str { + match self { + Severity::Error => "error", + Severity::Warning => "warning", + } + } +} + +/// One finding about the input, located in the source. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Diagnostic { + pub severity: Severity, + /// Stable identifier, e.g. `missing-token` or `unknown-command`. + pub code: &'static str, + pub message: String, + /// 1-based line. + pub line: usize, + /// 1-based column, counted in characters. + pub column: usize, + /// Byte range in the source. + pub start: usize, + pub end: usize, +} + +impl Diagnostic { + /// A diagnostic covering `node` of `source`. + pub fn at( + source: &str, + node: Node, + severity: Severity, + code: &'static str, + message: String, + ) -> Self { + Self::spanning(source, node, node.end_byte(), severity, code, message) + } + + /// A diagnostic from the start of `node` to byte `end`. + pub fn spanning( + source: &str, + node: Node, + end: usize, + severity: Severity, + code: &'static str, + message: String, + ) -> Self { + let mut columns = Columns::default(); + Self::counted(&mut columns, source, node, end, severity, code, message) + } + + /// [`spanning`](Self::spanning), counting its column with `columns`. + pub(crate) fn counted( + columns: &mut Columns, + source: &str, + node: Node, + end: usize, + severity: Severity, + code: &'static str, + message: String, + ) -> Self { + let start = node.start_byte(); + let point = node.start_position(); + let column = columns.column(source, start - point.column, start); + Self { + severity, + code, + message, + line: point.row + 1, + column, + start, + end, + } + } + + /// A diagnostic about the input as a whole, at its start. + pub fn whole(severity: Severity, code: &'static str, message: String) -> Self { + Self { + severity, + code, + message, + line: 1, + column: 1, + start: 0, + end: 0, + } + } + + pub fn is_error(&self) -> bool { + self.severity == Severity::Error + } +} + +/// Character columns of byte offsets, counted on from the last one on the +/// same line: diagnostics come mostly in source order, and counting each from +/// its line's start would be quadratic on a long line full of them. +#[derive(Debug, Default)] +pub(crate) struct Columns { + line_start: usize, + byte: usize, + chars: usize, +} + +impl Columns { + /// The 1-based character column of byte `start` on the line starting at + /// byte `line_start`. + fn column(&mut self, source: &str, line_start: usize, start: usize) -> usize { + if line_start != self.line_start || start < self.byte { + (self.line_start, self.byte, self.chars) = (line_start, line_start, 0); + } + self.chars += source + .get(self.byte..start) + .map_or(start - self.byte, |s| s.chars().count()); + self.byte = start; + self.chars + 1 + } +} + +impl fmt::Display for Diagnostic { + /// `3:12: error: missing '}' [missing-token]` + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}:{}: {}: {} [{}]", + self.line, + self.column, + self.severity.as_str(), + self.message, + self.code + ) + } +} + +/// Sort diagnostics by position and drop repeats (the reader walks some music +/// twice, e.g. a variable used inside `\relative`). +pub(crate) fn tidy(diagnostics: &mut Vec) { + diagnostics.sort_by(|a, b| { + (a.start, a.end, a.severity, a.code).cmp(&(b.start, b.end, b.severity, b.code)) + }); + diagnostics.dedup_by(|a, b| (a.start, a.end, a.code) == (b.start, b.end, b.code)); +} diff --git a/src/ir/direction.rs b/src/ir/direction.rs index 55fd4b4..9460d81 100644 --- a/src/ir/direction.rs +++ b/src/ir/direction.rs @@ -121,9 +121,43 @@ pub struct RehearsalMark { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct OctaveShift { pub shift_type: String, + /// MusicXML's size: the interval the shift spans, 8, 15 or 22 (0 for a + /// stop). pub size: i8, } +impl OctaveShift { + /// LilyPond's `\ottava #n`: `n` octaves up (positive) or down, at most + /// three either way (22ma); 0 stops the shift. + pub fn from_octaves(n: i64) -> Self { + let octaves = n.clamp(-3, 3) as i8; + let shift_type = match octaves.signum() { + 1 => "up", + -1 => "down", + _ => "stop", + }; + Self { + shift_type: shift_type.to_string(), + size: if octaves == 0 { + 0 + } else { + 7 * octaves.abs() + 1 + }, + } + } + + /// The octaves of the shift, as `\ottava` takes them: positive up, + /// negative down, 0 for a stop. + pub fn octaves(&self) -> i8 { + let n = (self.size.clamp(8, 22) - 1) / 7; + match self.shift_type.as_str() { + "up" => n, + "down" => -n, + _ => 0, + } + } +} + /// A pedal marking. /// /// From lytk-py's `PedalEvent`. diff --git a/src/ir/duration.rs b/src/ir/duration.rs index 4a2564b..a28433d 100644 --- a/src/ir/duration.rs +++ b/src/ir/duration.rs @@ -81,7 +81,11 @@ impl Duration { pub fn actual_duration(&self) -> Frac { // Dot formula: base * (2 - 1/2^dots) let dotted = self.base * dot_multiplier(self.dots); - // Tuplet scaling + // Tuplet scaling. No reader makes a 0 term (they drop such ratios); + // one from a hand-edited IR scales nothing rather than dividing by 0. + if self.tuplet_normal == 0 || self.tuplet_actual == 0 { + return dotted; + } dotted * Frac::new(self.tuplet_normal as i64, self.tuplet_actual as i64) } @@ -91,6 +95,10 @@ impl Duration { /// `1 / base` for standard durations. Returns `None` for non-power-of-two /// bases (breve, longa, maxima use 0, −1, −2 which we don't encode as ints). pub fn lilypond_log(&self) -> Option { + // A zero-length note (LilyPond's `c4*0`) has no written value. + if *self.base.numer() <= 0 { + return None; + } // log2(1/base) — e.g. quarter (1/4) → log2(4) = 2 let recip = self.base.recip(); if *recip.denom() != 1 { diff --git a/src/ir/interval.rs b/src/ir/interval.rs index 515f806..f3bf851 100644 --- a/src/ir/interval.rs +++ b/src/ir/interval.rs @@ -65,8 +65,10 @@ impl Interval { let number: i32 = num_str .parse() .map_err(|_| format!("interval `{s}` has an invalid number"))?; - if number < 1 { - return Err(format!("interval number must be ≥ 1 in `{s}`")); + // Past 99 (fourteen octaves) an interval is no longer music, and the + // semitone arithmetic below would overflow for large numbers. + if !(1..=99).contains(&number) { + return Err(format!("interval number must be between 1 and 99 in `{s}`")); } let (base, perfect) = base_semitones(number); let adjust = quality_adjust(qual, perfect) diff --git a/src/ir/lower/state.rs b/src/ir/lower/state.rs index c11bb3b..4f8ecc3 100644 --- a/src/ir/lower/state.rs +++ b/src/ir/lower/state.rs @@ -110,8 +110,10 @@ impl LowerState { idx } else { let idx = self.staves.len(); - self.staves - .push(StaffBuilder::new(String::new(), idx as u8 + 1)); + self.staves.push(StaffBuilder::new( + String::new(), + u8::try_from(idx + 1).unwrap_or(u8::MAX), + )); self.current_staff = Some(idx); idx } diff --git a/src/ir/lower/walk.rs b/src/ir/lower/walk.rs index 0d792be..089d515 100644 --- a/src/ir/lower/walk.rs +++ b/src/ir/lower/walk.rs @@ -22,7 +22,8 @@ pub(super) fn walk_music(music: &Music, state: &mut LowerState) { for (i, child) in children.iter().enumerate() { state.time = saved_time; - state.voice = saved_voice + i as u8; + // Voice numbers are `u8`: a 256th parallel branch shares the last. + state.voice = saved_voice.saturating_add(u8::try_from(i).unwrap_or(u8::MAX)); walk_music(child, state); if state.time > max_time { max_time = state.time; @@ -190,7 +191,7 @@ fn walk_context( let idx = state.staves.len(); state.staves.push(StaffBuilder::new( name.unwrap_or("").to_string(), - idx as u8 + 1, + u8::try_from(idx + 1).unwrap_or(u8::MAX), )); let saved_staff = state.current_staff; state.current_staff = Some(idx); diff --git a/src/ir/measure.rs b/src/ir/measure.rs index 0b9105b..6e7536a 100644 --- a/src/ir/measure.rs +++ b/src/ir/measure.rs @@ -113,13 +113,16 @@ impl TimeSignature { /// Total beats as a rational, handling compound signatures like "3+2". /// /// From lytk-py's `TimeSignature.beats_fraction` property. + /// + /// Total for any value: no reader makes a 0 denominator, and one from a + /// hand-edited IR counts as 1 rather than dividing by 0; beats saturate. pub fn beats_fraction(&self) -> Ratio { let numerator: i64 = self .beats .split('+') .filter_map(|b| b.trim().parse::().ok()) - .sum(); - Ratio::new(numerator, self.beat_type as i64) + .fold(0, i64::saturating_add); + Ratio::new(numerator, i64::from(self.beat_type.max(1))) } } diff --git a/src/ir/music.rs b/src/ir/music.rs index 3c4e459..b0ec452 100644 --- a/src/ir/music.rs +++ b/src/ir/music.rs @@ -308,6 +308,32 @@ impl Music { } } + /// [`written_length`](Self::written_length) as a float, which cannot + /// overflow: for the readers' length bound. + pub fn approx_length(&self) -> f64 { + let frac = |d: &Duration| { + let d = d.actual_duration(); + *d.numer() as f64 / *d.denom() as f64 + }; + match self { + Music::Note { duration, .. } + | Music::Chord { duration, .. } + | Music::Rest { duration, .. } + | Music::Skip { duration } => frac(duration), + Music::Sequential(items) => items.iter().map(Music::approx_length).sum(), + Music::Simultaneous(items) => { + items.iter().map(Music::approx_length).fold(0.0, f64::max) + } + Music::Tuplet { content, .. } + | Music::Context { content, .. } + | Music::Variable { content, .. } => content.approx_length(), + Music::Repeat { + body, alternatives, .. + } => body.approx_length() + alternatives.iter().map(Music::approx_length).sum::(), + _ => 0.0, + } + } + /// The duration of this music element (for leaf events). /// Returns None for containers — use `total_duration` for those. pub fn leaf_duration(&self) -> Option<&Duration> { diff --git a/src/ir/note.rs b/src/ir/note.rs index 787a3d3..4e5fc68 100644 --- a/src/ir/note.rs +++ b/src/ir/note.rs @@ -26,6 +26,9 @@ pub enum ArpeggioType { NonArpeggio, } +/// Most tremolo slashes a note carries (LilyPond's `:1024` on a whole note). +pub const MAX_TREMOLO_MARKS: u8 = 10; + /// A single pitched note. /// /// From lytk-py's `Note(IRNode)`. @@ -64,7 +67,8 @@ pub struct Note { pub stem_direction: String, pub notehead: String, pub print_object: bool, - /// Number of tremolo slashes (1–4) for single-note tremolo. + /// Number of tremolo slashes (1–4) for single-note tremolo; writers cap + /// it at [`MAX_TREMOLO_MARKS`]. pub tremolo_marks: u8, /// Whether this note is part of a two-note tremolo (paired with next/prev note). pub two_note_tremolo: bool, diff --git a/src/ir/timeline.rs b/src/ir/timeline.rs index 1e01edd..c790557 100644 --- a/src/ir/timeline.rs +++ b/src/ir/timeline.rs @@ -79,6 +79,11 @@ impl Timeline { } /// Position after the last element or event. + /// Elements and events held (what copying this timeline costs). + pub(crate) fn element_count(&self) -> usize { + self.lanes.values().map(Vec::len).sum::() + self.events.len() + } + pub(crate) fn end(&self) -> Frac { let lanes = self .lanes @@ -392,6 +397,12 @@ fn element_staff(e: &VoiceElement) -> u8 { // --------------------------------------------------------------------------- /// Score-wide bar layout. +/// Most bars a grid holds. A bar line is cut per `bar_len` of music, so a +/// tiny bar (`\set Score.measureLength = #(ly:make-moment 1 4294967295)`) or +/// an enormous piece would otherwise make billions of them. The grid stops +/// cutting there, and readers refuse a score whose grid reaches it. +pub(crate) const MAX_BARS: usize = 100_000; + pub(crate) struct Grid { /// `[start, end)` of each bar, contiguous from 0. pub(crate) bars: Vec<(Frac, Frac)>, @@ -488,7 +499,7 @@ impl Grid { b += bar_len; } } - while b < to { + while b < to && cuts.len() <= MAX_BARS { cuts.push(b); b += bar_len; } @@ -509,7 +520,11 @@ impl Grid { anchor = p; if let Some(ts) = ts { if meters.last().is_none_or(|(q, t)| *q != p || *t != ts) { - meters.retain(|(q, _)| *q != p); + // The controls come in position order, so a meter + // already set here can only be the last one. + if meters.last().is_some_and(|(q, _)| *q == p) { + meters.pop(); + } meters.push((p, ts)); } } @@ -540,7 +555,14 @@ impl Grid { if let Some(from) = free_from { free_spans.push((from, end.max(from))); } else { - grid_lines(&mut cuts, anchor, bar_len, at, end); + // Unless notes are tied on (`to_end`), bar lines past the last + // start are dropped below: stop cutting at the bar holding it. + let to = if to_end { + end + } else { + num::CheckedAdd::checked_add(&last_start, &bar_len).map_or(end, |t| end.min(t)) + }; + grid_lines(&mut cuts, anchor, bar_len, at, to); } cuts.retain(|&c| c <= end && (to_end || c == zero() || c <= last_start)); @@ -551,9 +573,16 @@ impl Grid { if end > last || bars.is_empty() { bars.push((last, end.max(last))); } + // Both in position order: one sweep. + let mut spans = free_spans.iter().peekable(); let senza = bars .iter() - .map(|(s, _)| free_spans.iter().any(|(a, b)| s >= a && s < b)) + .map(|(s, _)| { + while spans.peek().is_some_and(|(_, b)| b <= s) { + spans.next(); + } + spans.peek().is_some_and(|(a, b)| s >= a && s < b) + }) .collect(); Grid { bars, diff --git a/src/lib.rs b/src/lib.rs index 7b578fd..891e6cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ //! - [`ir`] — the IR: a measure-based [`Score`] (Layer 2) and a recursive //! [`MusicDocument`] music tree (Layer 1), bridged by lift/lower. //! - [`adapters`] — per-format readers/writers ([`ToIrAdapter`]/[`FromIrAdapter`]). +//! - [`diagnostics`] — what a reader found wrong with its input. //! - [`transforms`] — composable IR passes. //! - [`representations`] — note-array / event-sequence / piano-roll encoders. //! @@ -29,6 +30,7 @@ //! [`MusicDocument`]. pub mod adapters; +pub mod diagnostics; pub mod ir; pub mod parser; pub mod representations; diff --git a/src/lytk/__init__.py b/src/lytk/__init__.py index 8fe3a0b..09fdd26 100644 --- a/src/lytk/__init__.py +++ b/src/lytk/__init__.py @@ -4,15 +4,22 @@ from lytk._core import ( Chord, + Diagnostic, + HeaderField, + InternalError, + LilyPondSyntaxError, + LytkError, Measure, MusicDocument, Note, + ParseError, Part, Pitch, Rest, Score, Voice, change_language, + check_lilypond, compute_metrics, flatten, from_abc, @@ -31,6 +38,7 @@ from_musicxml_string, from_note_array, from_piano_roll, + header_fields, invert, retrograde, to_abc, @@ -46,8 +54,18 @@ transpose_interval, transpose_to_key, ) +from lytk._core import __version__ # noqa: F401 - the crate's version __all__ = [ + # Errors and diagnostics + "LytkError", + "ParseError", + "LilyPondSyntaxError", + "InternalError", + "Diagnostic", + "check_lilypond", + "HeaderField", + "header_fields", "MusicDocument", "Score", # Structured note navigation (read-only Layer-2 tree) diff --git a/src/lytk/_core.pyi b/src/lytk/_core.pyi index d3e6ee2..d586fa9 100644 --- a/src/lytk/_core.pyi +++ b/src/lytk/_core.pyi @@ -4,6 +4,73 @@ from __future__ import annotations from typing import Any, TypeVar +__version__: str + +class LytkError(Exception): + """Base class of the errors lytk raises itself.""" + +class InternalError(LytkError): + """A bug in lytk: the Rust code panicked. Every function and method that + reads, writes or transforms music raises this instead of letting a Rust + panic through; the message names the panic and where it happened.""" + +class ParseError(LytkError, ValueError): + """The input could not be read: malformed, not the format expected, or + past the reader's bounds. A ValueError too.""" + +class LilyPondSyntaxError(ParseError): + """LilyPond read with ``strict=True`` has errors.""" + + diagnostics: list[Diagnostic] + """Every diagnostic of the reading, errors and warnings.""" + +class Diagnostic: + """One finding of the LilyPond reader. Comparable, hashable, picklable.""" + + def __init__( + self, + severity: str, + code: str, + message: str, + line: int, + column: int, + start: int, + end: int, + ) -> None: ... + + @property + def severity(self) -> str: + """``"error"``: LilyPond rejects the input; ``"warning"``: lytk does not + read it, or cannot represent it, and reads around it.""" + ... + @property + def code(self) -> str: + """A stable identifier: ``syntax-error``, ``missing-token``, + ``invalid-duration``, ``invalid-ratio``, ``not-lilypond``, + ``too-large`` (errors); ``unknown-command``, ``unrecognized-token``, + ``ignored-include``, ``unknown-language``, ``dropped-music``, + ``skipped-score``, ``unsupported-value`` (warnings).""" + ... + @property + def message(self) -> str: ... + @property + def line(self) -> int: + """1-based.""" + ... + @property + def column(self) -> int: + """1-based, in characters.""" + ... + @property + def start(self) -> int: + """Character offset: ``text[d.start:d.end]`` is the span.""" + ... + @property + def end(self) -> int: ... + def __str__(self) -> str: + """``3:12: error: missing `}` [missing-token]``""" + ... + class Score: """Opaque handle to a parsed music score.""" @@ -23,6 +90,17 @@ class Score: def num_parts(self) -> int: ... @property def parts(self) -> list[str]: ... + @property + def diagnostics(self) -> list[Diagnostic]: + """What reading LilyPond reported (empty for other sources); not part + of ``to_dict``/``to_json``.""" + ... + @property + def header(self) -> dict[str, str]: + """Every header field: ``title``, ``subtitle``, ``composer``, + ``arranger`` and ``lyricist`` (LilyPond's ``poet``) when set, then the + others (``copyright``, ``opus``, ``texidoc``, …) by key.""" + ... def to_json(self) -> str: ... @staticmethod def from_json(json: str) -> Score: ... @@ -53,6 +131,16 @@ class MusicDocument: def arranger(self) -> str | None: ... @property def language(self) -> str | None: ... + @property + def diagnostics(self) -> list[Diagnostic]: + """What reading LilyPond reported (empty for other sources).""" + ... + @property + def lyricist(self) -> str | None: ... + @property + def header(self) -> dict[str, str]: + """Every header field, as :attr:`Score.header`.""" + ... def notes(self, resolution: int = 480) -> list[tuple[int, int, int, int]]: """Notes as ``(onset, duration, pitch, velocity)`` tuples in time steps.""" ... @@ -223,22 +311,74 @@ class Part: def __repr__(self) -> str: ... # -- Adapters ---------------------------------------------------------------- +# +# Readers (``from_*``, ``Score.from_json``/``from_dict``, +# ``MusicDocument.from_json``, ``flatten``) raise ParseError (a ValueError) +# when the input cannot be read, and OSError when a file cannot be opened. +# ``strict=True`` LilyPond readers raise LilyPondSyntaxError (a ParseError) on +# errors. Writers (``to_*``) raise ValueError when a score cannot be written, +# OSError when the file cannot be. Anything may raise InternalError. + +class HeaderField: + """A field of a LilyPond ``\\header`` block (see :func:`header_fields`).""" + + @property + def key(self) -> str: ... + @property + def value(self) -> str: + """The value as text: strings decoded, ``\\markup`` as its words, + ``#"…"`` as its string.""" + ... + @property + def start(self) -> int: + """Character offset: ``text[f.start:f.end]`` is the whole ``key = value``.""" + ... + @property + def end(self) -> int: ... + @property + def score(self) -> int | None: + """Index of the ``\\score`` block the field is in (file order), or + ``None`` at the top level.""" + ... + +def header_fields(text: str) -> list[HeaderField]: + """Every ``\\header`` field of LilyPond text, from the parse tree alone + (nothing is read); fields with values other than text (``##f``) are left + out.""" + ... + +def check_lilypond(text: str, *, semantic: bool = False) -> list[Diagnostic]: + """Diagnostics of LilyPond text, in source order: the syntax only (fast, + nothing is read), or with ``semantic=True`` what a reading reports too. + Never raises for bad input: input too large to read is a ``too-large`` + error.""" + ... def from_musicxml(path: str) -> Score: ... def from_musicxml_string(xml: str) -> Score: ... def from_musicxml_bytes(data: bytes) -> Score: """Parse MusicXML or compressed MXL from in-memory bytes.""" ... -def from_lilypond(path: str, *, language: str | None = None) -> Score: ... -def from_lilypond_string(text: str, *, language: str | None = None) -> Score: ... -def from_lilypond_movements(path: str, *, language: str | None = None) -> list[Score]: - """Every movement of a LilyPond file: one score per ``\\score`` block.""" +def from_lilypond( + path: str, *, language: str | None = None, strict: bool = False +) -> Score: + """The first movement; ``strict=True`` raises LilyPondSyntaxError on an + error. The diagnostics are in ``Score.diagnostics`` either way.""" + ... +def from_lilypond_string( + text: str, *, language: str | None = None, strict: bool = False +) -> Score: ... +def from_lilypond_movements( + path: str, *, language: str | None = None, strict: bool = False +) -> list[Score]: + """Every movement of a LilyPond file: one score per ``\\score`` block and + per top-level music expression, each with the file's diagnostics.""" ... def from_lilypond_music( - path: str, *, language: str | None = None + path: str, *, language: str | None = None, strict: bool = False ) -> MusicDocument: ... def from_lilypond_music_string( - text: str, *, language: str | None = None + text: str, *, language: str | None = None, strict: bool = False ) -> MusicDocument: ... def to_lilypond( score: Score, diff --git a/src/lytk/cli.py b/src/lytk/cli.py index 5388e00..f99a4f7 100644 --- a/src/lytk/cli.py +++ b/src/lytk/cli.py @@ -141,8 +141,8 @@ def read_score(path: str, forced: Format | str | None = None) -> lytk.Score: def _read_movements(path: str, forced: Format | None) -> list[lytk.Score]: - """Every movement: one per ``\\score`` block of a LilyPond file, one per - tune of an ABC file.""" + """Every movement: one per ``\\score`` block or top-level music expression + of a LilyPond file, one per tune of an ABC file.""" if path == "-": return [read_score(path, forced)] fmt = _input_format(path, forced) @@ -240,13 +240,7 @@ def run(*args: Any, **kwargs: Any) -> None: def _version(value: bool) -> None: if value: - try: - from importlib.metadata import version - - v = version("lytk") - except Exception: # noqa: BLE001 - not installed as a distribution - v = "unknown" - typer.echo(f"lytk {v}") + typer.echo(f"lytk {lytk.__version__}") raise typer.Exit() @@ -713,6 +707,40 @@ def batch( raise CliError(f"{failed} of {len(results)} job(s) failed") +# --------------------------------------------------------------------------- +# check +# --------------------------------------------------------------------------- + + +@_command() +def check( + files: Annotated[list[str], typer.Argument(help="LilyPond files (`-` for stdin).", show_default=False)], + semantic: Annotated[ + bool, + typer.Option("--semantic", help="Read the files too, and report what the reader drops or cannot represent."), + ] = False, + as_json: Annotated[bool, typer.Option("--json", help="Write the diagnostics as a JSON list.")] = False, +) -> None: + """Report the syntax errors of LilyPond files (and, with --semantic, what a + reading reports), one per line as FILE:LINE:COLUMN: SEVERITY: MESSAGE [CODE]. + + Exits with 1 when a file has an error; warnings alone exit with 0. + """ + found = [] + for name in files: + text = sys.stdin.buffer.read().decode("utf-8") if name == "-" else Path(name).read_text(encoding="utf-8") + found += [(name, d) for d in lytk.check_lilypond(text, semantic=semantic)] + if as_json: + keys = ("severity", "code", "message", "line", "column", "start", "end") + rows = [{"file": name, **{k: getattr(d, k) for k in keys}} for name, d in found] + write_bytes("-", (json.dumps(rows, indent=2, ensure_ascii=False) + "\n").encode("utf-8")) + else: + for name, d in found: + print(f"{name}:{d}") + if any(d.severity == "error" for _, d in found): + raise typer.Exit(1) + + # --------------------------------------------------------------------------- # flatten # --------------------------------------------------------------------------- diff --git a/src/parser.rs b/src/parser.rs index 96fa70c..ffa3d8b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -97,7 +97,17 @@ impl LilyPondParser { /// /// Rejects input nested deeper than [`MAX_NESTING_DEPTH`] (which would risk a /// stack-overflow abort during the IR walk) with [`ParseError::TooDeeplyNested`]. + /// A byte-order mark is whitespace wherever it stands, as in LilyPond. pub fn parse(&mut self, source: &str) -> Result { + // LilyPond reads a UTF-8 byte-order mark anywhere as whitespace; the + // grammar allows one only at the start. Three spaces keep every offset. + let spaced; + let source = if source.contains('\u{feff}') { + spaced = source.replace('\u{feff}', " "); + spaced.as_str() + } else { + source + }; let tree = self .parser .parse(source, None) @@ -177,6 +187,13 @@ mod tests { assert!(parser.parse(shallow).is_ok()); } + #[test] + fn test_byte_order_mark_is_whitespace_anywhere() { + let mut parser = LilyPondParser::new().unwrap(); + let tree = parser.parse("{ c'4 \u{feff}d'4 }").unwrap(); + assert!(!tree.root_node().has_error()); + } + #[test] fn test_parse_empty() { let mut parser = LilyPondParser::new().unwrap(); diff --git a/src/python.rs b/src/python.rs index 466d58b..e66aea4 100644 --- a/src/python.rs +++ b/src/python.rs @@ -1,21 +1,511 @@ //! PyO3 bindings — the `lytk._core` extension module. //! -//! Compiled only with the `python` feature (which maturin enables when building -//! wheels). The Rust crate proper carries no pyo3/numpy dependency. +//! Every function and method here that reads, writes or transforms music runs +//! inside [`guard`], so a Rust panic reaches Python as `lytk.InternalError`. use std::path::Path; +use std::borrow::Cow; + use pyo3::exceptions::{PyIOError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyModule}; +use pyo3::sync::GILOnceCell; +use pyo3::types::{PyBytes, PyDict, PyModule, PyTuple, PyType}; -use crate::adapters::{FromIrAdapter, FromMusicAdapter, ToIrAdapter, ToMusicAdapter}; +use crate::adapters::{FromIrAdapter, FromMusicAdapter, ToIrAdapter}; +use crate::diagnostics::Diagnostic; use crate::ir::interval::Interval; use crate::ir::language::{PitchLanguage, PitchMode}; use crate::ir::music::MusicDocument; use crate::ir::pitch::{Alter, Pitch, PitchStep}; use crate::ir::Score; use crate::{adapters, ir, navigation, representations, transforms}; + +// --------------------------------------------------------------------------- +// Errors and the panic firewall +// --------------------------------------------------------------------------- + +/// `create_exception!` in pyo3 0.22 checks a `gil-refs` feature that this +/// crate does not declare. +#[allow(unexpected_cfgs)] +mod errors { + use pyo3::create_exception; + + create_exception!( + lytk, + LytkError, + pyo3::exceptions::PyException, + "Base class of the errors lytk raises itself." + ); + create_exception!( + lytk, + InternalError, + LytkError, + "A bug in lytk: the Rust code panicked. The input is not to blame; please \ + report it at https://github.com/CSCPadova/lytk/issues." + ); +} +use errors::{InternalError, LytkError}; + +/// `ParseError(LytkError, ValueError)` and `LilyPondSyntaxError(ParseError)`: +/// classes with two bases, which `create_exception!` cannot make. +static PARSE_ERROR: GILOnceCell> = GILOnceCell::new(); +static SYNTAX_ERROR: GILOnceCell> = GILOnceCell::new(); + +fn new_exception( + py: Python<'_>, + name: &str, + bases: Vec>, + doc: &str, +) -> PyResult> { + let namespace = PyDict::new_bound(py); + namespace.set_item("__module__", "lytk")?; + namespace.set_item("__doc__", doc)?; + let class = + py.get_type_bound::() + .call1((name, PyTuple::new_bound(py, bases), namespace))?; + Ok(class.downcast_into::()?.unbind()) +} + +fn parse_error_type(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + let class = PARSE_ERROR.get_or_try_init(py, || { + new_exception( + py, + "ParseError", + vec![ + py.get_type_bound::(), + py.get_type_bound::(), + ], + "The input could not be read: malformed, not the format expected, or \ + past the reader's bounds. A ValueError too.", + ) + })?; + Ok(class.bind(py)) +} + +fn syntax_error_type(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + let class = SYNTAX_ERROR.get_or_try_init(py, || { + new_exception( + py, + "LilyPondSyntaxError", + vec![parse_error_type(py)?.clone()], + "LilyPond read with strict=True has errors. Its ``diagnostics`` \ + attribute lists them, with the warnings.", + ) + })?; + Ok(class.bind(py)) +} + +/// A `ParseError`: the input could not be read. +fn parse_error(message: String) -> PyErr { + Python::with_gil(|py| match parse_error_type(py) { + Ok(class) => PyErr::from_type_bound(class.clone(), message), + Err(e) => e, + }) +} + +/// A reader's failure: I/O stays `OSError`, anything else is a `ParseError`. +fn read_err(e: adapters::AdapterError) -> PyErr { + match e { + adapters::AdapterError::Io(_) => PyIOError::new_err(e.to_string()), + _ => parse_error(e.to_string()), + } +} + +/// How many `guard` calls are active, on any thread: the LilyPond reader walks +/// on a thread of its own, so a panic can happen away from the caller's. +static GUARDED: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); +/// The last guarded panic: message and source location. +static LAST_PANIC: std::sync::Mutex> = std::sync::Mutex::new(None); + +fn last_panic() -> std::sync::MutexGuard<'static, Option> { + LAST_PANIC + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()) +} + +/// Install (once) a panic hook that records a panic raised inside [`guard`] +/// instead of printing it to stderr; any other panic keeps the default hook. +fn install_panic_hook() { + static HOOK: std::sync::Once = std::sync::Once::new(); + HOOK.call_once(|| { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + if GUARDED.load(std::sync::atomic::Ordering::SeqCst) == 0 { + default(info); + return; + } + let location = info + .location() + .map(|l| format!(" (at {}:{})", l.file(), l.line())) + .unwrap_or_default(); + let message = format!("{}{location}", panic_message(info.payload())); + *last_panic() = Some(message); + })); + }); +} + +/// Run a binding's body so that a Rust panic becomes `lytk.InternalError` +/// instead of reaching Python as a `PanicException` (a `BaseException` that +/// `except Exception` does not catch) with its message on stderr. +fn guard(f: impl FnOnce() -> PyResult) -> PyResult { + use std::sync::atomic::Ordering::SeqCst; + GUARDED.fetch_add(1, SeqCst); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + GUARDED.fetch_sub(1, SeqCst); + result.unwrap_or_else(|payload| { + let detail = last_panic() + .take() + .unwrap_or_else(|| panic_message(&*payload)); + Err(InternalError::new_err(format!( + "lytk panicked: {detail}. This is a bug in lytk, not in the input; please \ + report it at https://github.com/CSCPadova/lytk/issues" + ))) + }) +} + +/// Deserialize a hand-supplied IR (`from_json`, `from_dict`), refusing values +/// no reader produces and the IR divides by: a 0 time-signature denominator +/// or tuplet term, or a negative duration. +fn ir_from_json(json: &str) -> PyResult { + let value: serde_json::Value = + serde_json::from_str(json).map_err(|e| parse_error(e.to_string()))?; + check_ir_values(&value).map_err(parse_error)?; + serde_json::from_value(value).map_err(|e| parse_error(e.to_string())) +} + +fn check_ir_values(value: &serde_json::Value) -> Result<(), String> { + use serde_json::Value; + // A duration's `[numerator, denominator]`: the readers keep both far below + // 2^40 (multipliers are bounded at 2^24), and the arithmetic relies on it. + const DURATION_TERM_MAX: i64 = 1 << 40; + let ratio = |v: &Value| Some((v.get(0)?.as_i64()?, v.get(1)?.as_i64()?)); + let bad = |what: String| Err(format!("invalid IR: {what}")); + match value { + Value::Object(map) => { + for (key, v) in map { + match key.as_str() { + "beat_type" | "tuplet_actual" | "tuplet_normal" | "actual" | "normal" + if v.as_u64() == Some(0) => + { + return bad(format!("{key} is 0")); + } + "base" => match ratio(v) { + Some((n, d)) + if (0..=DURATION_TERM_MAX).contains(&n) + && (1..=DURATION_TERM_MAX).contains(&d) => {} + Some(_) => return bad(format!("duration {v} out of range")), + None => check_ir_values(v)?, + }, + // The LilyPond reader's bound (its regression tests climb to 22). + "octave" if v.as_i64().is_some_and(|o| !(-128..=127).contains(&o)) => { + return bad(format!("octave {v} out of range -128..=127")); + } + // A pitch's alteration is a `[n, d]` ratio, a chord root's a number. + "alter" => { + let ok = match ratio(v) { + Some((n, d)) => d > 0 && n.abs() <= 4 * d, + None => v.as_f64().is_none_or(|x| x.abs() <= 4.0), + }; + if !ok { + return bad(format!("alteration {v} out of range")); + } + } + "beats" + if v.as_str().is_some_and(|b| { + b.split('+').count() > 64 + || b.split('+') + .any(|n| n.trim().parse::().is_ok_and(|n| n > 10_000)) + }) => + { + return bad(format!("time signature beats {v} out of range")); + } + _ => check_ir_values(v)?, + } + } + Ok(()) + } + Value::Array(items) => items.iter().try_for_each(check_ir_values), + _ => Ok(()), + } +} + +/// Panic on purpose, inside the firewall: lets the test suite check that a +/// panic becomes `InternalError` without printing anything. Not public API. +#[pyfunction] +fn _panic_for_tests(message: &str) -> PyResult<()> { + guard(|| panic!("{message}")) +} + +// --------------------------------------------------------------------------- +// Diagnostics +// --------------------------------------------------------------------------- + +/// One finding of the LilyPond reader: ``severity`` is ``"error"`` (LilyPond +/// rejects the input) or ``"warning"`` (lytk does not read it, or cannot +/// represent it); ``code`` is a stable identifier such as +/// ``"missing-token"``. ``line`` and ``column`` count from 1 (the column in +/// characters); ``text[start:end]`` is the span. +#[pyclass(name = "Diagnostic", module = "lytk", frozen, get_all, eq, hash)] +#[derive(Clone, PartialEq, Eq, Hash)] +struct PyDiagnostic { + severity: String, + code: String, + message: String, + line: usize, + column: usize, + start: usize, + end: usize, +} + +type DiagnosticFields = (String, String, String, usize, usize, usize, usize); + +#[pymethods] +impl PyDiagnostic { + #[new] + fn new( + severity: String, + code: String, + message: String, + line: usize, + column: usize, + start: usize, + end: usize, + ) -> Self { + Self { + severity, + code, + message, + line, + column, + start, + end, + } + } + + /// Pickling (an exception carrying diagnostics crosses process pools). + fn __reduce__<'py>(slf: &Bound<'py, Self>) -> (Bound<'py, PyType>, DiagnosticFields) { + let d = slf.get(); + ( + slf.get_type(), + ( + d.severity.clone(), + d.code.clone(), + d.message.clone(), + d.line, + d.column, + d.start, + d.end, + ), + ) + } + + /// ``3:12: error: missing `}` [missing-token]`` + fn __str__(&self) -> String { + format!( + "{}:{}: {}: {} [{}]", + self.line, self.column, self.severity, self.message, self.code + ) + } + + fn __repr__(&self) -> String { + format!("", self.__str__()) + } +} + +/// The character offset into `text` of each byte offset in `offsets` +/// (floored to a character boundary), counted in one pass over `text`. +fn char_offsets<'t>( + text: &'t str, + offsets: impl IntoIterator, +) -> impl Fn(usize) -> usize + 't { + let boundary = move |b: usize| { + let mut b = b.min(text.len()); + while !text.is_char_boundary(b) { + b -= 1; + } + b + }; + let mut bytes: Vec = offsets.into_iter().map(boundary).collect(); + bytes.sort_unstable(); + bytes.dedup(); + let (mut chars, mut count, mut at) = (Vec::with_capacity(bytes.len()), 0, 0); + for &b in &bytes { + count += text[at..b].chars().count(); + at = b; + chars.push(count); + } + move |b| chars[bytes.binary_search(&boundary(b)).unwrap_or(0)] +} + +/// Diagnostics for Python, their byte ranges turned into character offsets +/// into `text`. +fn py_diagnostics(text: &str, diagnostics: &[Diagnostic]) -> Vec { + let char_at = char_offsets(text, diagnostics.iter().flat_map(|d| [d.start, d.end])); + diagnostics + .iter() + .map(|d| PyDiagnostic { + severity: d.severity.as_str().to_string(), + code: d.code.to_string(), + message: d.message.clone(), + line: d.line, + column: d.column, + start: char_at(d.start), + end: char_at(d.end), + }) + .collect() +} + +/// With `strict`, an error among `diagnostics` fails the reading: a +/// `LilyPondSyntaxError` that carries them all. +fn check_strict(strict: bool, diagnostics: &[PyDiagnostic]) -> PyResult<()> { + let mut errors = diagnostics.iter().filter(|d| d.severity == "error"); + let (Some(first), true) = (errors.next(), strict) else { + return Ok(()); + }; + let more = match errors.count() { + 0 => String::new(), + 1 => " (and 1 more error)".to_string(), + n => format!(" (and {n} more errors)"), + }; + Python::with_gil(|py| { + let class = syntax_error_type(py)?.clone(); + let err = PyErr::from_type_bound(class, format!("{}{more}", first.__str__())); + err.value_bound(py) + .setattr("diagnostics", diagnostics.to_vec().into_py(py))?; + Err(err) + }) +} + +/// A field of a LilyPond ``\header`` block, from :func:`header_fields`: +/// ``text[start:end]`` is the whole ``key = value``, and ``score`` the index +/// of the ``\score`` block it is in (in file order), or *None*. +#[pyclass(name = "HeaderField", module = "lytk", frozen, get_all, eq, hash)] +#[derive(Clone, PartialEq, Eq, Hash)] +struct PyHeaderField { + key: String, + value: String, + start: usize, + end: usize, + score: Option, +} + +#[pymethods] +impl PyHeaderField { + fn __repr__(&self) -> String { + format!( + "HeaderField({:?}, {:?}, start={}, end={}, score={:?})", + self.key, self.value, self.start, self.end, self.score + ) + } +} + +/// Every ``\header`` field of LilyPond text, in source order, from the parse +/// tree alone (nothing is read). Values are text: strings decoded, a +/// ``\markup`` value as its plain words, ``#"…"`` as its string; fields with +/// other values (``##f``) are left out. ``text[f.start:f.end]`` is the whole +/// assignment, so a field can be read and cut. +#[pyfunction] +fn header_fields(py: Python<'_>, text: &str) -> PyResult> { + guard(|| { + let fields = py.allow_threads(|| adapters::ly_to_ir::header_fields(text)); + let char_at = char_offsets(text, fields.iter().flat_map(|f| [f.start, f.end])); + Ok(fields + .into_iter() + .map(|f| PyHeaderField { + start: char_at(f.start), + end: char_at(f.end), + key: f.key, + value: f.value, + score: f.score, + }) + .collect()) + }) +} + +/// The header fields of `meta` as a dict: the ones the IR names, then the +/// others sorted by key. +fn header_dict<'py>( + py: Python<'py>, + meta: &ir::score::ScoreMetadata, +) -> PyResult> { + let dict = PyDict::new_bound(py); + let named = [ + ("title", &meta.title), + ("subtitle", &meta.subtitle), + ("composer", &meta.composer), + ("arranger", &meta.arranger), + ("lyricist", &meta.lyricist), + ]; + for (key, value) in named { + if let Some(value) = value { + dict.set_item(key, value)?; + } + } + let mut extra: Vec<(&String, &String)> = meta.extra.iter().collect(); + extra.sort(); + for (key, value) in extra { + dict.set_item(key, value)?; + } + Ok(dict) +} + +/// Read LilyPond, from a file or text, releasing the GIL: the text and the +/// reading. +fn read_lilypond( + py: Python<'_>, + source: Result<&str, &str>, + language: Option<&str>, +) -> PyResult<(String, adapters::ly_to_ir::LyReading)> { + let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); + if let Some(lang_str) = language { + adapter = adapter.with_language(parse_language(lang_str)?); + } + py.allow_threads(|| { + let text = match source { + Ok(path) => Cow::Owned(adapters::ly_to_ir::read_source(Path::new(path))?), + Err(text) => Cow::Borrowed(text), + }; + let reading = adapter.read_str(&text)?; + Ok((text.into_owned(), reading)) + }) + .map_err(read_err) +} + +/// The first movement of a LilyPond reading, with its diagnostics. +fn first_movement( + text: &str, + reading: adapters::ly_to_ir::LyReading, + strict: bool, +) -> PyResult<(Score, Vec)> { + let (score, diagnostics) = reading.into_first(); + let diagnostics = py_diagnostics(text, &diagnostics); + check_strict(strict, &diagnostics)?; + Ok((score, diagnostics)) +} + +/// Check LilyPond text and return its :class:`Diagnostic` list, errors and +/// warnings in source order. By default only the syntax, from the parse tree +/// (fast, nothing is read); with ``semantic=True`` also what a reading +/// reports: invalid durations and ratios, unknown commands, input it does not +/// read. Input too large to read is an error here, not an exception. +#[pyfunction] +#[pyo3(signature = (text, *, semantic=false))] +fn check_lilypond(py: Python<'_>, text: &str, semantic: bool) -> PyResult> { + guard(|| { + let diagnostics = py.allow_threads(|| adapters::ly_to_ir::check(text, semantic)); + Ok(py_diagnostics(text, &diagnostics)) + }) +} + // --------------------------------------------------------------------------- // PyScore — opaque wrapper for the IR Score // --------------------------------------------------------------------------- @@ -29,10 +519,27 @@ use crate::{adapters, ir, navigation, representations, transforms}; #[derive(Clone)] struct PyScore { inner: Score, + diagnostics: Vec, +} + +impl From for PyScore { + fn from(inner: Score) -> Self { + Self { + inner, + diagnostics: Vec::new(), + } + } } #[pymethods] impl PyScore { + /// What reading LilyPond reported, as :class:`Diagnostic` objects (empty + /// for other sources). Not part of :meth:`to_dict` or :meth:`to_json`. + #[getter] + fn diagnostics(&self) -> Vec { + self.diagnostics.clone() + } + /// Score title (from the MusicXML or LilyPond header). #[getter] fn title(&self) -> Option { @@ -63,6 +570,14 @@ impl PyScore { self.inner.metadata.lyricist.clone() } + /// Every header field as a dict of strings: ``title``, ``subtitle``, + /// ``composer``, ``arranger`` and ``lyricist`` (LilyPond's ``poet``) when + /// set, then the others (``copyright``, ``opus``, ``texidoc``, …) by key. + #[getter] + fn header<'py>(&self, py: Python<'py>) -> PyResult> { + header_dict(py, &self.inner.metadata) + } + /// Active LilyPond pitch language (e.g. ``"nederlands"``), or *None*. #[getter] fn language(&self) -> Option { @@ -98,52 +613,57 @@ impl PyScore { /// objects, walkable as ``part.measures → measure.voices → voice.elements`` /// (each element a :class:`Note` / :class:`Rest` / :class:`Chord`). A /// structure-preserving complement to the flat :meth:`notes` tuples. - fn iter_parts(&self) -> Vec { - self.inner - .parts() - .iter() - .map(|p| navigation::PyPart::from_ir(p)) - .collect() + fn iter_parts(&self) -> PyResult> { + guard(|| { + Ok({ + self.inner + .parts() + .iter() + .map(|p| navigation::PyPart::from_ir(p)) + .collect() + }) + }) } /// Serialize the full score IR to a JSON string. fn to_json(&self) -> PyResult { - serde_json::to_string_pretty(&self.inner).map_err(|e| PyValueError::new_err(e.to_string())) + guard(|| { + serde_json::to_string_pretty(&self.inner) + .map_err(|e| PyValueError::new_err(e.to_string())) + }) } /// Deserialize a score from a JSON string. #[staticmethod] fn from_json(json: &str) -> PyResult { - let score: Score = - serde_json::from_str(json).map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyScore { inner: score }) + guard(|| Ok(PyScore::from(ir_from_json::(json)?))) } /// Serialize the score IR to a Python dict. fn to_dict<'py>(&self, py: Python<'py>) -> PyResult> { - let json_str = - serde_json::to_string(&self.inner).map_err(|e| PyValueError::new_err(e.to_string()))?; - let json_mod = PyModule::import_bound(py, "json")?; - json_mod.call_method1("loads", (json_str,)) + guard(|| { + let json_str = serde_json::to_string(&self.inner) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + let json_mod = PyModule::import_bound(py, "json")?; + json_mod.call_method1("loads", (json_str,)) + }) } /// Deserialize a score from a Python dict. #[staticmethod] fn from_dict(dict: &Bound<'_, PyAny>) -> PyResult { - let py = dict.py(); - let json_mod = PyModule::import_bound(py, "json")?; - let json_str: String = json_mod.call_method1("dumps", (dict,))?.extract()?; - let score: Score = - serde_json::from_str(&json_str).map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyScore { inner: score }) + guard(|| { + let py = dict.py(); + let json_mod = PyModule::import_bound(py, "json")?; + let json_str: String = json_mod.call_method1("dumps", (dict,))?.extract()?; + Ok(PyScore::from(ir_from_json::(&json_str)?)) + }) } /// Lift this measure-based :class:`Score` to a Layer-1 :class:`MusicDocument` /// (the form the ML representations consume). - fn to_music_document(&self) -> PyMusicDocument { - PyMusicDocument { - inner: ir::lift::lift_to_music(&self.inner), - } + fn to_music_document(&self) -> PyResult { + guard(|| Ok(PyMusicDocument::from(ir::lift::lift_to_music(&self.inner)))) } /// The score's notes as ``(onset, duration, pitch, velocity)`` tuples in time @@ -151,13 +671,17 @@ impl PyScore { /// way to iterate notes directly, without going through /// :func:`to_note_array` or hand-walking :meth:`to_dict`. #[pyo3(signature = (resolution = representations::note_array::DEFAULT_RESOLUTION))] - fn notes(&self, resolution: u16) -> Vec<(u32, u32, u8, u8)> { - let doc = ir::lift::lift_to_music(&self.inner); - representations::to_note_array(&doc, resolution) - .notes - .iter() - .map(|n| (n.onset, n.duration, n.pitch, n.velocity)) - .collect() + fn notes(&self, resolution: u16) -> PyResult> { + guard(|| { + Ok({ + let doc = ir::lift::lift_to_music(&self.inner); + representations::to_note_array(&doc, resolution) + .notes + .iter() + .map(|n| (n.onset, n.duration, n.pitch, n.velocity)) + .collect() + }) + }) } fn __repr__(&self) -> String { @@ -186,10 +710,27 @@ impl PyScore { #[derive(Clone)] struct PyMusicDocument { inner: MusicDocument, + diagnostics: Vec, +} + +impl From for PyMusicDocument { + fn from(inner: MusicDocument) -> Self { + Self { + inner, + diagnostics: Vec::new(), + } + } } #[pymethods] impl PyMusicDocument { + /// What reading LilyPond reported, as :class:`Diagnostic` objects (empty + /// for other sources). Not part of :meth:`to_json`. + #[getter] + fn diagnostics(&self) -> Vec { + self.diagnostics.clone() + } + /// Score title (from the header). #[getter] fn title(&self) -> Option { @@ -214,6 +755,18 @@ impl PyMusicDocument { self.inner.metadata.arranger.clone() } + /// Lyricist (LilyPond ``poet``/``lyricist``). + #[getter] + fn lyricist(&self) -> Option { + self.inner.metadata.lyricist.clone() + } + + /// Every header field as a dict of strings (see :attr:`Score.header`). + #[getter] + fn header<'py>(&self, py: Python<'py>) -> PyResult> { + header_dict(py, &self.inner.metadata) + } + /// Active LilyPond pitch language (e.g. ``"nederlands"``), or *None*. #[getter] fn language(&self) -> Option { @@ -227,32 +780,35 @@ impl PyMusicDocument { /// time steps (``resolution`` = steps per quarter note) — a lightweight, /// numpy-free way to iterate notes directly. #[pyo3(signature = (resolution = representations::note_array::DEFAULT_RESOLUTION))] - fn notes(&self, resolution: u16) -> Vec<(u32, u32, u8, u8)> { - representations::to_note_array(&self.inner, resolution) - .notes - .iter() - .map(|n| (n.onset, n.duration, n.pitch, n.velocity)) - .collect() + fn notes(&self, resolution: u16) -> PyResult> { + guard(|| { + Ok({ + representations::to_note_array(&self.inner, resolution) + .notes + .iter() + .map(|n| (n.onset, n.duration, n.pitch, n.velocity)) + .collect() + }) + }) } /// Serialize the music document to a JSON string. fn to_json(&self) -> PyResult { - serde_json::to_string_pretty(&self.inner).map_err(|e| PyValueError::new_err(e.to_string())) + guard(|| { + serde_json::to_string_pretty(&self.inner) + .map_err(|e| PyValueError::new_err(e.to_string())) + }) } /// Deserialize a music document from a JSON string. #[staticmethod] fn from_json(json: &str) -> PyResult { - let doc: MusicDocument = - serde_json::from_str(json).map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyMusicDocument { inner: doc }) + guard(|| Ok(PyMusicDocument::from(ir_from_json::(json)?))) } /// Convert this music document to a measure-based :class:`Score`. - fn to_score(&self) -> PyScore { - PyScore { - inner: ir::lower::lower_to_score(&self.inner), - } + fn to_score(&self) -> PyResult { + guard(|| Ok(PyScore::from(ir::lower::lower_to_score(&self.inner)))) } fn __repr__(&self) -> String { @@ -268,12 +824,8 @@ impl PyMusicDocument { // Adapter functions // --------------------------------------------------------------------------- -/// Map an adapter error to the appropriate Python exception: an I/O failure -/// becomes `IOError`, every parse/validation failure becomes `ValueError`. -/// -/// Previously the file readers mapped *all* adapter errors to `IOError`, so a -/// malformed-but-readable file looked the same as a missing one — a batch loader -/// wrapping reads in `except IOError` would silently swallow corrupt files. +/// A writer's failure: I/O becomes `OSError`, anything else `ValueError`. +/// Readers use [`read_err`]. fn adapter_err(e: adapters::AdapterError) -> PyErr { let msg = e.to_string(); match e { @@ -286,21 +838,23 @@ fn adapter_err(e: adapters::AdapterError) -> PyErr { /// :class:`Score`. #[pyfunction] fn from_musicxml(py: Python<'_>, path: &str) -> PyResult { - let adapter = adapters::mxml_to_ir::MxmlToIrAdapter::new(); - let score = py - .allow_threads(|| adapter.convert_file(Path::new(path))) - .map_err(adapter_err)?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::mxml_to_ir::MxmlToIrAdapter::new(); + let score = py + .allow_threads(|| adapter.convert_file(Path::new(path))) + .map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Parse a MusicXML string into a :class:`Score`. #[pyfunction] fn from_musicxml_string(xml: &str) -> PyResult { - let adapter = adapters::mxml_to_ir::MxmlToIrAdapter::new(); - let score = adapter - .convert_str(xml) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::mxml_to_ir::MxmlToIrAdapter::new(); + let score = adapter.convert_str(xml).map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Resolve a pitch-language name, raising a `ValueError` for unknown names. @@ -309,85 +863,114 @@ fn parse_language(name: &str) -> PyResult { .ok_or_else(|| PyValueError::new_err(format!("unknown language: {name}"))) } -/// Parse a LilyPond (``.ly``) file into a :class:`Score`. +/// Parse a LilyPond (``.ly``) file into a :class:`Score` (its first movement). +/// +/// ``strict=True`` raises :class:`LilyPondSyntaxError` if the reading reports +/// an error; the diagnostics are in :attr:`Score.diagnostics` either way. #[pyfunction] -#[pyo3(signature = (path, *, language=None))] -fn from_lilypond(py: Python<'_>, path: &str, language: Option<&str>) -> PyResult { - let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); - if let Some(lang_str) = language { - adapter = adapter.with_language(parse_language(lang_str)?); - } - let score = py - .allow_threads(|| adapter.convert_file(Path::new(path))) - .map_err(adapter_err)?; - Ok(PyScore { inner: score }) +#[pyo3(signature = (path, *, language=None, strict=false))] +fn from_lilypond( + py: Python<'_>, + path: &str, + language: Option<&str>, + strict: bool, +) -> PyResult { + guard(|| { + let (text, reading) = read_lilypond(py, Ok(path), language)?; + let (score, diagnostics) = first_movement(&text, reading, strict)?; + Ok(PyScore { + inner: score, + diagnostics, + }) + }) } /// Parse every movement of a LilyPond file: one :class:`Score` per ``\\score`` -/// block (a file without ``\\score`` blocks is a single movement). +/// block and per top-level music expression, in order, as LilyPond makes a +/// score of each. Each score carries the file's diagnostics. #[pyfunction] -#[pyo3(signature = (path, *, language=None))] +#[pyo3(signature = (path, *, language=None, strict=false))] fn from_lilypond_movements( py: Python<'_>, path: &str, language: Option<&str>, + strict: bool, ) -> PyResult> { - let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); - if let Some(lang_str) = language { - adapter = adapter.with_language(parse_language(lang_str)?); - } - let scores = py - .allow_threads(|| adapter.convert_file_multi(Path::new(path))) - .map_err(adapter_err)?; - Ok(scores.into_iter().map(|inner| PyScore { inner }).collect()) + guard(|| { + let (text, reading) = read_lilypond(py, Ok(path), language)?; + let diagnostics = py_diagnostics(&text, &reading.diagnostics); + check_strict(strict, &diagnostics)?; + Ok(reading + .scores + .into_iter() + .map(|inner| PyScore { + inner, + diagnostics: diagnostics.clone(), + }) + .collect()) + }) } -/// Parse a LilyPond string into a :class:`Score`. +/// Parse a LilyPond string into a :class:`Score` (its first movement); see +/// :func:`from_lilypond`. #[pyfunction] -#[pyo3(signature = (text, *, language=None))] -fn from_lilypond_string(text: &str, language: Option<&str>) -> PyResult { - let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); - if let Some(lang_str) = language { - adapter = adapter.with_language(parse_language(lang_str)?); - } - let score = adapter - .convert_str(text) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyScore { inner: score }) +#[pyo3(signature = (text, *, language=None, strict=false))] +fn from_lilypond_string( + py: Python<'_>, + text: &str, + language: Option<&str>, + strict: bool, +) -> PyResult { + guard(|| { + let (text, reading) = read_lilypond(py, Err(text), language)?; + let (score, diagnostics) = first_movement(&text, reading, strict)?; + Ok(PyScore { + inner: score, + diagnostics, + }) + }) } /// Parse a LilyPond file into a :class:`MusicDocument` (Layer 1 Music tree). /// -/// This preserves structural information like contexts and simultaneous blocks. +/// This preserves structural information like contexts and simultaneous +/// blocks. ``strict`` as in :func:`from_lilypond`. #[pyfunction] -#[pyo3(signature = (path, *, language=None))] +#[pyo3(signature = (path, *, language=None, strict=false))] fn from_lilypond_music( py: Python<'_>, path: &str, language: Option<&str>, + strict: bool, ) -> PyResult { - let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); - if let Some(lang_str) = language { - adapter = adapter.with_language(parse_language(lang_str)?); - } - let doc = py - .allow_threads(|| adapter.convert_file_to_music(Path::new(path))) - .map_err(adapter_err)?; - Ok(PyMusicDocument { inner: doc }) + guard(|| { + let (text, reading) = read_lilypond(py, Ok(path), language)?; + let (score, diagnostics) = first_movement(&text, reading, strict)?; + Ok(PyMusicDocument { + inner: ir::lift::lift_to_music(&score), + diagnostics, + }) + }) } -/// Parse a LilyPond string into a :class:`MusicDocument` (Layer 1 Music tree). +/// Parse a LilyPond string into a :class:`MusicDocument` (Layer 1 Music +/// tree); see :func:`from_lilypond_music`. #[pyfunction] -#[pyo3(signature = (text, *, language=None))] -fn from_lilypond_music_string(text: &str, language: Option<&str>) -> PyResult { - let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); - if let Some(lang_str) = language { - adapter = adapter.with_language(parse_language(lang_str)?); - } - let doc = adapter - .convert_str_to_music(text) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyMusicDocument { inner: doc }) +#[pyo3(signature = (text, *, language=None, strict=false))] +fn from_lilypond_music_string( + py: Python<'_>, + text: &str, + language: Option<&str>, + strict: bool, +) -> PyResult { + guard(|| { + let (text, reading) = read_lilypond(py, Err(text), language)?; + let (score, diagnostics) = first_movement(&text, reading, strict)?; + Ok(PyMusicDocument { + inner: ir::lift::lift_to_music(&score), + diagnostics, + }) + }) } /// Emit a :class:`MusicDocument` as a LilyPond string. If *path* is given the @@ -395,14 +978,16 @@ fn from_lilypond_music_string(text: &str, language: Option<&str>) -> PyResult) -> PyResult { - let adapter = adapters::ir_to_ly::IrToLyAdapter::new(); - let output = adapter - .convert_music(&doc.inner) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(p) = path { - std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; - } - Ok(output) + guard(|| { + let adapter = adapters::ir_to_ly::IrToLyAdapter::new(); + let output = adapter + .convert_music(&doc.inner) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(p) = path { + std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; + } + Ok(output) + }) } /// Emit a :class:`Score` as a LilyPond string. If *path* is given the result @@ -418,29 +1003,31 @@ fn to_lilypond( language: Option<&str>, relative: Option, ) -> PyResult { - let mut adapter = adapters::ir_to_ly::IrToLyAdapter::new(); - if let Some(lang_str) = language { - adapter = adapter.with_language(parse_language(lang_str)?); - } else if let Some(lang) = score.inner.metadata.pitch_language { - adapter = adapter.with_language(lang); - } - let output = match relative { - None => adapter.convert(&score.inner), - Some(relative) => { - let mut score = score.inner.clone(); - score.metadata.pitch_mode = if relative { - PitchMode::Relative - } else { - PitchMode::Absolute - }; - adapter.convert(&score) + guard(|| { + let mut adapter = adapters::ir_to_ly::IrToLyAdapter::new(); + if let Some(lang_str) = language { + adapter = adapter.with_language(parse_language(lang_str)?); + } else if let Some(lang) = score.inner.metadata.pitch_language { + adapter = adapter.with_language(lang); } - } - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(p) = path { - std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; - } - Ok(output) + let output = match relative { + None => adapter.convert(&score.inner), + Some(relative) => { + let mut score = score.inner.clone(); + score.metadata.pitch_mode = if relative { + PitchMode::Relative + } else { + PitchMode::Absolute + }; + adapter.convert(&score) + } + } + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(p) = path { + std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; + } + Ok(output) + }) } /// Emit a :class:`Score` as a MusicXML string. If *path* is given the result @@ -448,21 +1035,23 @@ fn to_lilypond( #[pyfunction] #[pyo3(signature = (score, path=None))] fn to_musicxml(py: Python<'_>, score: &PyScore, path: Option<&str>) -> PyResult { - let adapter = adapters::ir_to_mxml::IrToMxmlAdapter::new(); - let output = py - .allow_threads(|| adapter.convert(&score.inner)) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(p) = path { - if p.to_ascii_lowercase().ends_with(".mxl") { - // Compressed MXL, not plain XML in a misnamed file. - adapter - .write(&score.inner, Path::new(p)) - .map_err(|e| PyIOError::new_err(e.to_string()))?; - } else { - std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; + guard(|| { + let adapter = adapters::ir_to_mxml::IrToMxmlAdapter::new(); + let output = py + .allow_threads(|| adapter.convert(&score.inner)) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(p) = path { + if p.to_ascii_lowercase().ends_with(".mxl") { + // Compressed MXL, not plain XML in a misnamed file. + adapter + .write(&score.inner, Path::new(p)) + .map_err(|e| PyIOError::new_err(e.to_string()))?; + } else { + std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; + } } - } - Ok(output) + Ok(output) + }) } /// Recursively expand ``\include`` directives in a LilyPond file, returning the @@ -479,49 +1068,55 @@ fn flatten( include_paths: Option>, add_markers: bool, ) -> PyResult { - let opts = adapters::ly_flatten::FlattenOpts { - include_paths: include_paths - .unwrap_or_default() - .into_iter() - .map(std::path::PathBuf::from) - .collect(), - add_markers, - }; - let text = adapters::ly_flatten::flatten(Path::new(input), opts) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(p) = output { - std::fs::write(p, &text).map_err(|e| PyIOError::new_err(e.to_string()))?; - } - Ok(text) + guard(|| { + let opts = adapters::ly_flatten::FlattenOpts { + include_paths: include_paths + .unwrap_or_default() + .into_iter() + .map(std::path::PathBuf::from) + .collect(), + add_markers, + }; + let text = adapters::ly_flatten::flatten(Path::new(input), opts) + .map_err(|e| parse_error(e.to_string()))?; + if let Some(p) = output { + std::fs::write(p, &text).map_err(|e| PyIOError::new_err(e.to_string()))?; + } + Ok(text) + }) } /// Parse an ABC notation (``.abc``) file into a :class:`Score`. #[pyfunction] fn from_abc(path: &str) -> PyResult { - let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); - let score = adapter.convert_file(Path::new(path)).map_err(adapter_err)?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); + let score = adapter.convert_file(Path::new(path)).map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Parse every tune of an ABC file: one :class:`Score` per ``X:`` tune (the /// text before the first ``X:`` applies to all of them). #[pyfunction] fn from_abc_tunes(py: Python<'_>, path: &str) -> PyResult> { - let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); - let scores = py - .allow_threads(|| adapter.convert_file_tunes(Path::new(path))) - .map_err(adapter_err)?; - Ok(scores.into_iter().map(|inner| PyScore { inner }).collect()) + guard(|| { + let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); + let scores = py + .allow_threads(|| adapter.convert_file_tunes(Path::new(path))) + .map_err(read_err)?; + Ok(scores.into_iter().map(PyScore::from).collect()) + }) } /// Parse an ABC notation string into a :class:`Score`. #[pyfunction] fn from_abc_string(text: &str) -> PyResult { - let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); - let score = adapter - .convert_str(text) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); + let score = adapter.convert_str(text).map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Emit a :class:`Score` as an ABC notation string. If *path* is given the @@ -530,35 +1125,39 @@ fn from_abc_string(text: &str) -> PyResult { #[pyfunction] #[pyo3(signature = (score, path=None))] fn to_abc(score: &PyScore, path: Option<&str>) -> PyResult { - let doc = ir::lift::lift_to_music(&score.inner); - let adapter = adapters::ir_to_abc::IrToAbcAdapter::new(); - let output = adapter - .convert_music(&doc) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(p) = path { - std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; - } - Ok(output) + guard(|| { + let doc = ir::lift::lift_to_music(&score.inner); + let adapter = adapters::ir_to_abc::IrToAbcAdapter::new(); + let output = adapter + .convert_music(&doc) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(p) = path { + std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; + } + Ok(output) + }) } /// Parse a Humdrum (``**kern``) file into a :class:`Score`. #[pyfunction] fn from_humdrum(py: Python<'_>, path: &str) -> PyResult { - let adapter = adapters::humdrum_to_ir::HumdrumToIrAdapter::new(); - let score = py - .allow_threads(|| adapter.convert_file(Path::new(path))) - .map_err(adapter_err)?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::humdrum_to_ir::HumdrumToIrAdapter::new(); + let score = py + .allow_threads(|| adapter.convert_file(Path::new(path))) + .map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Parse a Humdrum (``**kern``) string into a :class:`Score`. #[pyfunction] fn from_humdrum_string(text: &str) -> PyResult { - let adapter = adapters::humdrum_to_ir::HumdrumToIrAdapter::new(); - let score = adapter - .convert_str(text) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::humdrum_to_ir::HumdrumToIrAdapter::new(); + let score = adapter.convert_str(text).map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Emit a :class:`Score` as a Humdrum ``**kern`` string. If *path* is given @@ -566,14 +1165,16 @@ fn from_humdrum_string(text: &str) -> PyResult { #[pyfunction] #[pyo3(signature = (score, path=None))] fn to_humdrum(py: Python<'_>, score: &PyScore, path: Option<&str>) -> PyResult { - let adapter = adapters::ir_to_humdrum::IrToHumdrumAdapter::new(); - let output = py - .allow_threads(|| adapter.convert(&score.inner)) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(p) = path { - std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; - } - Ok(output) + guard(|| { + let adapter = adapters::ir_to_humdrum::IrToHumdrumAdapter::new(); + let output = py + .allow_threads(|| adapter.convert(&score.inner)) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(p) = path { + std::fs::write(p, &output).map_err(|e| PyIOError::new_err(e.to_string()))?; + } + Ok(output) + }) } /// Parse a Standard MIDI File into a :class:`Score`. ``quantize`` (4, 8, 16 @@ -583,8 +1184,10 @@ fn to_humdrum(py: Python<'_>, score: &PyScore, path: Option<&str>) -> PyResult, swing: Option) -> PyResult { - let bytes = std::fs::read(path).map_err(|e| PyIOError::new_err(e.to_string()))?; - from_midi_bytes(&bytes, quantize, swing) + guard(|| { + let bytes = std::fs::read(path).map_err(|e| PyIOError::new_err(e.to_string()))?; + from_midi_bytes(&bytes, quantize, swing) + }) } /// Parse a Standard MIDI File from in-memory ``bytes`` into a :class:`Score` @@ -592,20 +1195,24 @@ fn from_midi(path: &str, quantize: Option, swing: Option) -> PyResult #[pyfunction] #[pyo3(signature = (data, *, quantize=None, swing=None))] fn from_midi_bytes(data: &[u8], quantize: Option, swing: Option) -> PyResult { - let adapter = adapters::midi_to_ir::MidiToIrAdapter::new() - .with_quantize(quantize) - .with_swing(swing); - let score = adapter.convert_bytes(data).map_err(adapter_err)?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::midi_to_ir::MidiToIrAdapter::new() + .with_quantize(quantize) + .with_swing(swing); + let score = adapter.convert_bytes(data).map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Parse MusicXML or compressed MXL from in-memory ``bytes`` into a /// :class:`Score` (auto-detects `.mxl` vs plain XML; no temp file needed). #[pyfunction] fn from_musicxml_bytes(data: &[u8]) -> PyResult { - let adapter = adapters::mxml_to_ir::MxmlToIrAdapter::new(); - let score = adapter.convert_bytes(data).map_err(adapter_err)?; - Ok(PyScore { inner: score }) + guard(|| { + let adapter = adapters::mxml_to_ir::MxmlToIrAdapter::new(); + let score = adapter.convert_bytes(data).map_err(read_err)?; + Ok(PyScore::from(score)) + }) } /// Write a :class:`Score` to a Standard MIDI File. Repeats are played out @@ -613,22 +1220,27 @@ fn from_musicxml_bytes(data: &[u8]) -> PyResult { #[pyfunction] #[pyo3(signature = (score, path, *, unfold_repeats=true))] fn to_midi(score: &PyScore, path: &str, unfold_repeats: bool) -> PyResult<()> { - let adapter = adapters::ir_to_midi::IrToMidiAdapter::new().with_unfold_repeats(unfold_repeats); - adapter - .write(&score.inner, Path::new(path)) - .map_err(adapter_err)?; - Ok(()) + guard(|| { + let adapter = + adapters::ir_to_midi::IrToMidiAdapter::new().with_unfold_repeats(unfold_repeats); + adapter + .write(&score.inner, Path::new(path)) + .map_err(adapter_err)?; + Ok(()) + }) } /// Serialize a :class:`Score` to compressed MusicXML (``.mxl``) ``bytes`` — a /// ZIP archive, the in-memory counterpart of ``to_musicxml(score, "x.mxl")``. #[pyfunction] fn to_mxl_bytes<'py>(py: Python<'py>, score: &PyScore) -> PyResult> { - let adapter = adapters::ir_to_mxml::IrToMxmlAdapter::new(); - let bytes = py - .allow_threads(|| adapter.convert_mxl_bytes(&score.inner)) - .map_err(adapter_err)?; - Ok(PyBytes::new_bound(py, &bytes)) + guard(|| { + let adapter = adapters::ir_to_mxml::IrToMxmlAdapter::new(); + let bytes = py + .allow_threads(|| adapter.convert_mxl_bytes(&score.inner)) + .map_err(adapter_err)?; + Ok(PyBytes::new_bound(py, &bytes)) + }) } /// Serialize a :class:`Score` to Standard MIDI File ``bytes`` (the in-memory @@ -640,9 +1252,12 @@ fn to_midi_bytes<'py>( score: &PyScore, unfold_repeats: bool, ) -> PyResult> { - let adapter = adapters::ir_to_midi::IrToMidiAdapter::new().with_unfold_repeats(unfold_repeats); - let bytes = adapter.convert_bytes(&score.inner).map_err(adapter_err)?; - Ok(PyBytes::new_bound(py, &bytes)) + guard(|| { + let adapter = + adapters::ir_to_midi::IrToMidiAdapter::new().with_unfold_repeats(unfold_repeats); + let bytes = adapter.convert_bytes(&score.inner).map_err(adapter_err)?; + Ok(PyBytes::new_bound(py, &bytes)) + }) } // --------------------------------------------------------------------------- @@ -662,12 +1277,12 @@ fn dispatch_transform( if let Ok(s) = music.extract::>() { let score: &Score = &s.inner; let inner = py.allow_threads(|| on_score(score)); - return Ok(PyScore { inner }.into_py(py)); + return Ok(PyScore::from(inner).into_py(py)); } if let Ok(d) = music.extract::>() { let doc: &MusicDocument = &d.inner; let inner = py.allow_threads(|| on_music(doc)); - return Ok(PyMusicDocument { inner }.into_py(py)); + return Ok(PyMusicDocument::from(inner).into_py(py)); } Err(PyValueError::new_err( "expected a Score or MusicDocument as first argument", @@ -679,12 +1294,19 @@ fn dispatch_transform( /// of the same type; the original is not modified. #[pyfunction] fn transpose(py: Python<'_>, music: &Bound<'_, PyAny>, semitones: i32) -> PyResult { - dispatch_transform( - py, - music, - |s| transforms::transpose::transpose(s, semitones), - |d| transforms::transpose::transpose_music(d, semitones), - ) + guard(|| { + if !(-127..=127).contains(&semitones) { + return Err(PyValueError::new_err(format!( + "transpose by {semitones} semitones: at most 127 either way" + ))); + } + dispatch_transform( + py, + music, + |s| transforms::transpose::transpose(s, semitones), + |d| transforms::transpose::transpose_music(d, semitones), + ) + }) } /// Transpose all pitches by a named diatonic *interval* (e.g. ``"M3"``, ``"m3"``, @@ -697,13 +1319,15 @@ fn transpose_interval( music: &Bound<'_, PyAny>, interval: &str, ) -> PyResult { - let iv = Interval::from_name(interval).map_err(PyValueError::new_err)?; - dispatch_transform( - py, - music, - |s| transforms::transpose::transpose_interval(s, iv), - |d| transforms::transpose::transpose_interval_music(d, iv), - ) + guard(|| { + let iv = Interval::from_name(interval).map_err(PyValueError::new_err)?; + dispatch_transform( + py, + music, + |s| transforms::transpose::transpose_interval(s, iv), + |d| transforms::transpose::transpose_interval_music(d, iv), + ) + }) } /// Transpose so the piece's tonic becomes *key* (e.g. ``"D"``, ``"Bb"``, @@ -711,13 +1335,15 @@ fn transpose_interval( /// :class:`MusicDocument`; returns the same type. #[pyfunction] fn transpose_to_key(py: Python<'_>, music: &Bound<'_, PyAny>, key: &str) -> PyResult { - let tonic = _core_parse_tonic(key).map_err(PyValueError::new_err)?; - dispatch_transform( - py, - music, - |s| transforms::transpose::transpose_to_key(s, tonic), - |d| transforms::transpose::transpose_to_key_music(d, tonic), - ) + guard(|| { + let tonic = _core_parse_tonic(key).map_err(PyValueError::new_err)?; + dispatch_transform( + py, + music, + |s| transforms::transpose::transpose_to_key(s, tonic), + |d| transforms::transpose::transpose_to_key_music(d, tonic), + ) + }) } use crate::ir::pitch::parse_tonic as _core_parse_tonic; @@ -726,13 +1352,15 @@ use crate::ir::pitch::parse_tonic as _core_parse_tonic; /// Accepts a :class:`Score` or :class:`MusicDocument`; returns the same type. #[pyfunction] fn change_language(py: Python<'_>, music: &Bound<'_, PyAny>, language: &str) -> PyResult { - let lang = parse_language(language)?; - dispatch_transform( - py, - music, - |s| transforms::language::change_language(s, lang), - |d| transforms::language::change_language_music(d, lang), - ) + guard(|| { + let lang = parse_language(language)?; + dispatch_transform( + py, + music, + |s| transforms::language::change_language(s, lang), + |d| transforms::language::change_language_music(d, lang), + ) + }) } /// Invert intervals around an axis pitch. Returns a new :class:`Score`. @@ -754,27 +1382,36 @@ fn invert( alter: i32, octave: i32, ) -> PyResult { - let s = PitchStep::from_name(step) - .ok_or_else(|| PyValueError::new_err(format!("invalid step: {step}")))?; - let axis = Pitch::with_alter(s, Alter::from_integer(alter), octave); - dispatch_transform( - py, - music, - |s| transforms::invert::invert(s, axis), - |d| transforms::invert::invert_music(d, axis), - ) + guard(|| { + let s = PitchStep::from_name(step) + .ok_or_else(|| PyValueError::new_err(format!("invalid step: {step}")))?; + if !(-128..=127).contains(&octave) || !(-4..=4).contains(&alter) { + return Err(PyValueError::new_err(format!( + "invert axis out of range: octave {octave} (-128..=127), alter {alter} (-4..=4)" + ))); + } + let axis = Pitch::with_alter(s, Alter::from_integer(alter), octave); + dispatch_transform( + py, + music, + |s| transforms::invert::invert(s, axis), + |d| transforms::invert::invert_music(d, axis), + ) + }) } /// Reverse the music in time. Accepts a :class:`Score` or /// :class:`MusicDocument`; returns the same type. #[pyfunction] fn retrograde(py: Python<'_>, music: &Bound<'_, PyAny>) -> PyResult { - dispatch_transform( - py, - music, - transforms::retrograde::retrograde, - transforms::retrograde::retrograde_music, - ) + guard(|| { + dispatch_transform( + py, + music, + transforms::retrograde::retrograde, + transforms::retrograde::retrograde_music, + ) + }) } // --------------------------------------------------------------------------- @@ -796,17 +1433,21 @@ fn to_note_array<'py>( py: Python<'py>, doc: &PyMusicDocument, resolution: u16, -) -> Bound<'py, PyArray2> { - let arr = representations::to_note_array(&doc.inner, resolution); - let n = arr.notes.len(); - let mut data = Array2::::zeros((n, 4)); - for (i, row) in arr.notes.iter().enumerate() { - data[[i, 0]] = row.onset as i32; - data[[i, 1]] = row.duration as i32; - data[[i, 2]] = row.pitch as i32; - data[[i, 3]] = row.velocity as i32; - } - data.into_pyarray_bound(py) +) -> PyResult>> { + guard(|| { + Ok({ + let arr = representations::to_note_array(&doc.inner, resolution); + let n = arr.notes.len(); + let mut data = Array2::::zeros((n, 4)); + for (i, row) in arr.notes.iter().enumerate() { + data[[i, 0]] = row.onset as i32; + data[[i, 1]] = row.duration as i32; + data[[i, 2]] = row.pitch as i32; + data[[i, 3]] = row.velocity as i32; + } + data.into_pyarray_bound(py) + }) + }) } /// Decode a note-based array of shape ``(N, 4)`` back into a @@ -814,25 +1455,25 @@ fn to_note_array<'py>( #[pyfunction] #[pyo3(signature = (array, resolution=representations::note_array::DEFAULT_RESOLUTION))] fn from_note_array(array: PyReadonlyArray2, resolution: u16) -> PyResult { - let view = array.as_array(); - if view.ncols() != 4 { - return Err(PyValueError::new_err( - "note array must have shape (N, 4): (onset, duration, pitch, velocity)", - )); - } - let notes = view - .rows() - .into_iter() - .map(|r| NoteRow { - onset: r[0].max(0) as u32, - duration: r[1].max(0) as u32, - pitch: r[2].clamp(0, 127) as u8, - velocity: r[3].clamp(0, 127) as u8, - }) - .collect(); - let na = NoteArray { resolution, notes }; - Ok(PyMusicDocument { - inner: representations::from_note_array(&na), + guard(|| { + let view = array.as_array(); + if view.ncols() != 4 { + return Err(PyValueError::new_err( + "note array must have shape (N, 4): (onset, duration, pitch, velocity)", + )); + } + let notes = view + .rows() + .into_iter() + .map(|r| NoteRow { + onset: r[0].max(0) as u32, + duration: r[1].max(0) as u32, + pitch: r[2].clamp(0, 127) as u8, + velocity: r[3].clamp(0, 127) as u8, + }) + .collect(); + let na = NoteArray { resolution, notes }; + Ok(PyMusicDocument::from(representations::from_note_array(&na))) }) } @@ -853,16 +1494,20 @@ fn to_event_sequence<'py>( max_time_shift: u32, velocity_bins: u8, encode_velocity: bool, -) -> Bound<'py, PyArray1> { - let arr = representations::to_note_array(&doc.inner, resolution); - let opts = EventOptions { - max_time_shift, - velocity_bins, - encode_velocity, - }; - let seq = representations::to_event_sequence(&arr, &opts); - let codes: Vec = seq.codes.iter().map(|&c| c as i64).collect(); - Array1::from(codes).into_pyarray_bound(py) +) -> PyResult>> { + guard(|| { + Ok({ + let arr = representations::to_note_array(&doc.inner, resolution); + let opts = EventOptions { + max_time_shift, + velocity_bins, + encode_velocity, + }; + let seq = representations::to_event_sequence(&arr, &opts); + let codes: Vec = seq.codes.iter().map(|&c| c as i64).collect(); + Array1::from(codes).into_pyarray_bound(py) + }) + }) } /// Decode an event-sequence array back into a :class:`MusicDocument`. @@ -880,19 +1525,21 @@ fn from_event_sequence( max_time_shift: u32, velocity_bins: u8, encode_velocity: bool, -) -> PyMusicDocument { - let codes: Vec = array.as_array().iter().map(|&c| c.max(0) as u32).collect(); - let seq = EventSequence { - codes, - resolution, - max_time_shift, - velocity_bins, - encode_velocity, - }; - let na = representations::from_event_sequence(&seq); - PyMusicDocument { - inner: representations::from_note_array(&na), - } +) -> PyResult { + guard(|| { + Ok({ + let codes: Vec = array.as_array().iter().map(|&c| c.max(0) as u32).collect(); + let seq = EventSequence { + codes, + resolution, + max_time_shift, + velocity_bins, + encode_velocity, + }; + let na = representations::from_event_sequence(&seq); + PyMusicDocument::from(representations::from_note_array(&na)) + }) + }) } /// Encode a :class:`MusicDocument` as a piano-roll matrix of shape @@ -905,14 +1552,18 @@ fn to_piano_roll<'py>( doc: &PyMusicDocument, resolution: u16, encode_velocity: bool, -) -> Bound<'py, PyArray2> { - let arr = representations::to_note_array(&doc.inner, resolution); - let pr = representations::to_piano_roll(&arr, encode_velocity); - let t = pr.num_steps as usize; - // pr.data is already row-major (t, 128). - let data = Array2::from_shape_vec((t, PITCH_COUNT), pr.data) - .expect("piano-roll data length matches T*128"); - data.into_pyarray_bound(py) +) -> PyResult>> { + guard(|| { + Ok({ + let arr = representations::to_note_array(&doc.inner, resolution); + let pr = representations::to_piano_roll(&arr, encode_velocity); + let t = pr.num_steps as usize; + // pr.data is already row-major (t, 128). + let data = Array2::from_shape_vec((t, PITCH_COUNT), pr.data) + .expect("piano-roll data length matches T*128"); + data.into_pyarray_bound(py) + }) + }) } /// Decode a piano-roll matrix of shape ``(T, 128)`` back into a @@ -924,21 +1575,21 @@ fn from_piano_roll( resolution: u16, encode_velocity: bool, ) -> PyResult { - let view = array.as_array(); - if view.ncols() != PITCH_COUNT { - return Err(PyValueError::new_err("piano roll must have shape (T, 128)")); - } - let num_steps = view.nrows() as u32; - let data: Vec = view.iter().copied().collect(); - let pr = PianoRoll { - resolution, - num_steps, - encode_velocity, - data, - }; - let na = representations::from_piano_roll(&pr); - Ok(PyMusicDocument { - inner: representations::from_note_array(&na), + guard(|| { + let view = array.as_array(); + if view.ncols() != PITCH_COUNT { + return Err(PyValueError::new_err("piano roll must have shape (T, 128)")); + } + let num_steps = view.nrows() as u32; + let data: Vec = view.iter().copied().collect(); + let pr = PianoRoll { + resolution, + num_steps, + encode_velocity, + data, + }; + let na = representations::from_piano_roll(&pr); + Ok(PyMusicDocument::from(representations::from_note_array(&na))) }) } @@ -960,26 +1611,28 @@ fn compute_metrics<'py>( resolution: u16, measure_resolution: Option, ) -> PyResult> { - use representations::metrics as mx; - let arr = representations::to_note_array(&doc.inner, resolution); - let mr = measure_resolution.unwrap_or(4 * resolution as u32).max(1); - - let dict = pyo3::types::PyDict::new_bound(py); - dict.set_item("n_pitches_used", mx::n_pitches_used(&arr))?; - dict.set_item("n_pitch_classes_used", mx::n_pitch_classes_used(&arr))?; - dict.set_item("pitch_range", mx::pitch_range(&arr))?; - dict.set_item( - "pitch_class_histogram", - mx::pitch_class_histogram(&arr).to_vec(), - )?; - dict.set_item("pitch_entropy", mx::pitch_entropy(&arr))?; - dict.set_item("pitch_class_entropy", mx::pitch_class_entropy(&arr))?; - dict.set_item("polyphony", mx::polyphony(&arr))?; - dict.set_item("polyphony_rate", mx::polyphony_rate(&arr, 2))?; - dict.set_item("empty_beat_rate", mx::empty_beat_rate(&arr))?; - dict.set_item("scale_consistency", mx::scale_consistency(&arr))?; - dict.set_item("groove_consistency", mx::groove_consistency(&arr, mr))?; - Ok(dict) + guard(|| { + use representations::metrics as mx; + let arr = representations::to_note_array(&doc.inner, resolution); + let mr = measure_resolution.unwrap_or(4 * resolution as u32).max(1); + + let dict = pyo3::types::PyDict::new_bound(py); + dict.set_item("n_pitches_used", mx::n_pitches_used(&arr))?; + dict.set_item("n_pitch_classes_used", mx::n_pitch_classes_used(&arr))?; + dict.set_item("pitch_range", mx::pitch_range(&arr))?; + dict.set_item( + "pitch_class_histogram", + mx::pitch_class_histogram(&arr).to_vec(), + )?; + dict.set_item("pitch_entropy", mx::pitch_entropy(&arr))?; + dict.set_item("pitch_class_entropy", mx::pitch_class_entropy(&arr))?; + dict.set_item("polyphony", mx::polyphony(&arr))?; + dict.set_item("polyphony_rate", mx::polyphony_rate(&arr, 2))?; + dict.set_item("empty_beat_rate", mx::empty_beat_rate(&arr))?; + dict.set_item("scale_consistency", mx::scale_consistency(&arr))?; + dict.set_item("groove_consistency", mx::groove_consistency(&arr, mr))?; + Ok(dict) + }) } // --------------------------------------------------------------------------- @@ -989,6 +1642,18 @@ fn compute_metrics<'py>( /// The compiled Rust extension module (``lytk._core``). #[pymodule] fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { + install_panic_hook(); + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + m.add("LytkError", m.py().get_type_bound::())?; + m.add("InternalError", m.py().get_type_bound::())?; + m.add("ParseError", parse_error_type(m.py())?)?; + m.add("LilyPondSyntaxError", syntax_error_type(m.py())?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(check_lilypond, m)?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(header_fields, m)?)?; + m.add_function(wrap_pyfunction!(_panic_for_tests, m)?)?; + // Score class m.add_class::()?; m.add_class::()?; diff --git a/src/representations/note_array.rs b/src/representations/note_array.rs index 936d5b3..1b41fd0 100644 --- a/src/representations/note_array.rs +++ b/src/representations/note_array.rs @@ -76,12 +76,12 @@ struct FlatNote { /// Convert a whole-note `Frac` time to integer time steps for the given /// resolution (time steps per quarter note). Rounds to nearest. fn frac_to_steps(t: Frac, resolution: u16) -> u32 { - // quarter = 1/4 whole, so steps = t * 4 * resolution. - let scaled = t * Frac::from_integer(4 * resolution as i64); - let num = *scaled.numer(); - let den = *scaled.denom(); - // Rounded integer division for non-negative values. - (((num * 2 + den) / (den * 2)).max(0)) as u32 + // quarter = 1/4 whole, so steps = t * 4 * resolution, computed in i128 so + // no numerator or denominator the IR holds can overflow it. + let num = i128::from(*t.numer()) * 4 * i128::from(resolution); + let den = i128::from(*t.denom()); + // Rounded integer division for non-negative values, saturating at u32. + ((num * 2 + den) / (den * 2)).clamp(0, i128::from(u32::MAX)) as u32 } /// Inverse of [`frac_to_steps`]. diff --git a/src/tree-sitter/README.md b/src/tree-sitter/README.md index 27ba8b0..db42f4a 100644 --- a/src/tree-sitter/README.md +++ b/src/tree-sitter/README.md @@ -8,4 +8,11 @@ network fetch. `bindings/rust/build.rs` compiles `src/parser.c` and MIT licensed — see [`LICENSE`](LICENSE). Do not edit these files by hand; to update, regenerate them upstream and copy the result in. +Vendored from upstream commit +[`b3b38a6`](https://github.com/nwhetsell/tree-sitter-lilypond/commit/b3b38a6645255115048d6a1069b070811b477249) +(2025-12-12, parsers generated with Tree-sitter 0.26.2): `lilypond/src` → +`src/`, `lilypond-scheme/src` → `lilypond-scheme/src/`. Upstream's grammar has +not changed since (checked 2026-09-27 against `3d999a1`, a regeneration with +Tree-sitter 0.27.0). + [upstream]: https://github.com/nwhetsell/tree-sitter-lilypond diff --git a/tests/fidelity.rs b/tests/fidelity.rs index 3ca97f0..8c0b9e9 100644 --- a/tests/fidelity.rs +++ b/tests/fidelity.rs @@ -128,7 +128,11 @@ const MIDI_DUR_BASELINE: usize = 4; // Bar structure kept (bar count and each bar's length, score-wide; see // `common::bar_lengths`). Note signatures merge ties and ignore bar lines, so // they can't see a note that moved into the wrong bar. Measured 2026-09-25. -const LY_BARS_BASELINE: usize = 27; +// Epic J3 (2026-09-27): two PDMX fixtures write `fis4 -\markup \bold a8 d4`; +// LilyPond reads `a8` as the markup's word (its bar check then fails), and so +// does the reader now, where it used to read the note a8. Their bars are short +// and the LY round trip does not keep them: 27 → 26. +const LY_BARS_BASELINE: usize = 26; const XML_BARS_BASELINE: usize = 152; const ABC_BARS_BASELINE: usize = 4; const XML_ABC_BARS_BASELINE: usize = 142; diff --git a/tests/fuzz_inputs.rs b/tests/fuzz_inputs.rs index 40abb28..e885a14 100644 --- a/tests/fuzz_inputs.rs +++ b/tests/fuzz_inputs.rs @@ -9,10 +9,18 @@ use proptest::prelude::*; use _core::adapters::abc_to_ir::AbcToIrAdapter; use _core::adapters::humdrum_to_ir::HumdrumToIrAdapter; -use _core::adapters::ly_to_ir::LyToIrAdapter; +use _core::adapters::ir_to_abc::IrToAbcAdapter; +use _core::adapters::ir_to_humdrum::IrToHumdrumAdapter; +use _core::adapters::ir_to_ly::IrToLyAdapter; +use _core::adapters::ir_to_midi::IrToMidiAdapter; +use _core::adapters::ir_to_mxml::IrToMxmlAdapter; +use _core::adapters::ly_to_ir::{check, LyToIrAdapter}; use _core::adapters::midi_to_ir::MidiToIrAdapter; use _core::adapters::mxml_to_ir::MxmlToIrAdapter; -use _core::adapters::ToIrAdapter; +use _core::adapters::{FromIrAdapter, FromMusicAdapter, ToIrAdapter, ToMusicAdapter}; +use _core::ir::language::PitchLanguage; +use _core::representations; +use _core::transforms; proptest! { #![proptest_config(ProptestConfig::with_cases(128))] @@ -64,6 +72,150 @@ proptest! { } } +// --- LilyPond-shaped fuzz ----------------------------------------------------- +// +// Arbitrary text almost never reaches the reader's number handling: `\time 0/0` +// survived 128 random-text cases per run for months. These properties build +// LilyPond out of real constructs whose numbers are edge values. + +/// Numbers at the edges of every integer type the reader casts to. +fn edge_number() -> impl Strategy { + prop_oneof![ + 4 => (0u64..=16).prop_map(|n| n.to_string()), + 1 => Just("0".to_string()), + 1 => Just("255".to_string()), + 1 => Just("256".to_string()), + 1 => Just("65535".to_string()), + 1 => Just("65536".to_string()), + 1 => Just("2147483648".to_string()), + 1 => Just("4294967296".to_string()), + 1 => Just("18446744073709551615".to_string()), + 1 => Just("99999999999999999999999999".to_string()), + ] +} + +/// One LilyPond construct with edge numbers in its holes. +fn construct() -> BoxedStrategy { + let n = edge_number; + prop_oneof![ + (n(), n()).prop_map(|(a, b)| format!("\\time {a}/{b} c4 d e f")), + (n(), n(), n()).prop_map(|(a, b, c)| format!("\\time {a}+{b}/{c} c8 d e")), + (n(), n()).prop_map(|(a, b)| format!("\\tuplet {a}/{b} {{ c8 d e }}")), + (n(), n()).prop_map(|(a, b)| format!("\\times {a}/{b} {{ c8 d e }}")), + (n(), n(), n(), n()).prop_map(|(a, b, c, d)| { + format!("\\tuplet {a}/{b} {{ c8 \\tuplet {c}/{d} {{ d16 e f }} }}") + }), + n().prop_map(|a| format!("\\partial {a} c4 d e f")), + (n(), n()).prop_map(|(a, b)| format!("\\partial {a}*{b} c4")), + n().prop_map(|a| format!("c{a} d4")), + (n(), 0usize..300).prop_map(|(a, k)| format!("c{a}{} d4", ".".repeat(k))), + (n(), n()).prop_map(|(a, b)| format!("c4*{a} d{b}")), + (n(), n(), n()).prop_map(|(a, b, c)| format!("c{a}*{b}/{c} d4")), + (n(), n()).prop_map(|(a, b)| format!("r{a}*{b} R1*{b} c4")), + (n(), n()).prop_map(|(a, b)| format!("s{a}*{b} c4")), + (n(), n()).prop_map(|(a, b)| format!("\\skip {a}*{b} c4")), + (n(), n()).prop_map(|(a, b)| { + format!("\\set Score.measureLength = #(ly:make-moment {a} {b}) c4 d e f") + }), + (n(), n()).prop_map(|(a, b)| { + format!("\\set Timing.baseMoment = #(ly:make-moment {a}/{b}) c4") + }), + n().prop_map(|a| format!("\\repeat unfold {a} {{ c4 d }}")), + (n(), n()).prop_map(|(a, b)| { + format!("\\repeat unfold {a} {{ \\repeat unfold {b} {{ c16 }} }}") + }), + n().prop_map(|a| { + format!("\\repeat volta {a} {{ c4 }} \\alternative {{ {{ d4 }} {{ e4 }} }}") + }), + (n(), n()).prop_map(|(a, b)| format!("\\scaleDurations {a}/{b} {{ c4 d }}")), + n().prop_map(|a| format!("\\grace {{ c{a} d }} e4")), + n().prop_map(|a| format!("\\ottava #{a} c4")), + (n(), n()).prop_map(|(a, b)| format!("\\tempo {a} = {b} c4")), + (n(), n()).prop_map(|(a, b)| format!("\\tempo 4 = {a}-{b} c4")), + n().prop_map(|a| format!("\\clef \"treble_{a}\" c4")), + n().prop_map(|a| format!("\\mark {a} c4")), + n().prop_map(|a| format!("{a} d4")), + (n(), n()).prop_map(|(a, b)| format!("\\compoundMeter #'(({a} {b} 8)) c8 d e")), + (n(), n(), 0usize..300) + .prop_map(|(a, b, k)| { format!("\\figuremode {{ <{a}>{b}{} }}", ".".repeat(k)) }), + (n(), n()).prop_map(|(a, b)| format!("\\figuremode {{ <6>{a}*{b} }}")), + (n(), n()).prop_map(|(a, b)| format!("\\chordmode {{ c{a}:{b} }}")), + (n(), n()).prop_map(|(a, b)| format!("\\drummode {{ bd{a}*{b} sn }}")), + n().prop_map(|a| format!("c{} d4", "'".repeat(a.len().min(40) * 8))), + ] + .boxed() +} + +/// A score built from a few constructs, in a staff or as bare music. +fn lilypond_program() -> impl Strategy { + (proptest::collection::vec(construct(), 1..4), any::()).prop_map(|(parts, staff)| { + let music = parts.join(" "); + if staff { + format!("\\score {{ \\new Staff {{ {music} }} \\layout {{ }} }}") + } else { + format!("{{ {music} }}") + } + }) +} + +/// Cases per LilyPond-shaped property: few in `cargo test` (inputs at the +/// reader's bounds take seconds in debug builds), many in CI's fuzz job +/// (`LYTK_FUZZ_CASES=1024`). +fn fuzz_cases(default: u32) -> u32 { + std::env::var("LYTK_FUZZ_CASES") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(fuzz_cases(32)))] + + /// LilyPond-shaped input must not crash any reader entry point. + #[test] + fn ly_reader_survives_lilypond_shaped_input(src in lilypond_program()) { + let reader = LyToIrAdapter::new(); + let _ = reader.convert_str(&src); + let _ = reader.convert_str_multi(&src); + let _ = reader.convert_str_to_music(&src); + let _ = LyToIrAdapter::new() + .with_language(PitchLanguage::English) + .convert_str(&src); + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(fuzz_cases(16)))] + + /// Whatever the reader makes of LilyPond-shaped input, every writer, + /// transform and representation must handle it. (A piano roll is only + /// built for short music, at a coarse resolution: it is dense by nature, + /// and this property is about panics, not allocation.) + #[test] + fn downstream_survives_lilypond_shaped_input(src in lilypond_program()) { + let Ok(score) = LyToIrAdapter::new().convert_str(&src) else { return Ok(()) }; + // Long music (up to the reader's 100,000 bars) makes big documents, not + // new panics: keep the property's memory modest. + if score.parts().iter().map(|p| p.measures.len()).sum::() > 5_000 { + return Ok(()); + } + let doc = _core::ir::lift::lift_to_music(&score); + let _ = IrToLyAdapter::new().convert(&score); + let _ = IrToLyAdapter::new().convert_music(&doc); + let _ = IrToMxmlAdapter::new().convert(&score); + let _ = IrToMidiAdapter::new().convert_bytes(&score); + let _ = IrToAbcAdapter::new().convert_music(&doc); + let _ = IrToHumdrumAdapter::new().convert(&score); + let _ = transforms::transpose::transpose(&score, 7); + let _ = transforms::retrograde::retrograde(&score); + let _ = _core::ir::lower::lower_to_score(&doc); + let arr = representations::to_note_array(&doc, 4); + if arr.length() <= 100_000 { + let _ = representations::to_piano_roll(&arr, true); + } + } +} + // --- Targeted regression inputs the audit reproduced as crashes/hangs -------- #[test] @@ -101,3 +253,344 @@ fn empty_and_garbage_inputs_are_clean_errors() { let _ = midi.convert_bytes(input.as_bytes()); } } + +/// LilyPond inputs that panicked before 0.3.0 (Epic J1), each through every +/// reader entry point. Each construct is dropped (a time signature, a tuplet +/// ratio, a measure length) and the music around it is still read. +const LY_PANICKED: &[&str] = &[ + // `\time N/0`: a zero denominator reached `TimeSignature::beats_fraction` + r"{ \time 3/0 c4 d e f }", + // `\time 3/256`: the denominator was truncated to a `u8` 0 + r"{ \time 3/256 c4 d e f }", + r"{ \time 3+2/512 c8 d e f g }", + // `\tuplet 0/N` and `\times N/0`: a zero ratio term reached `actual_duration` + r"{ \tuplet 0/2 { c8 d e } }", + r"{ \times 2/0 { c8 d e } }", + r"{ \tuplet 3/2 { c8 \tuplet 0/1 { d16 e } f8 } }", + // `ly:make-moment N 0`: a zero denominator reached `Frac::new` + r"{ \set Score.measureLength = #(ly:make-moment 3 0) c4 d e f }", + r"{ \set Score.measureLength = #(ly:make-moment 3/0) c4 d e f }", +]; + +fn read_everywhere(src: &str) -> _core::ir::score::Score { + let reader = LyToIrAdapter::new(); + let _ = reader.convert_str_multi(src).expect("reads"); + let _ = reader.convert_str_to_music(src).expect("reads"); + reader.convert_str(src).expect("reads") +} + +#[test] +fn lilypond_inputs_that_panicked_are_read() { + for src in LY_PANICKED { + let score = read_everywhere(src); + let json = serde_json::to_string(&score).unwrap(); + assert!(!json.contains("\"beat_type\":0"), "{src}: {json}"); + assert!(!json.contains("\"tuplet_actual\":0"), "{src}"); + assert!(!json.contains("\"tuplet_normal\":0"), "{src}"); + } +} + +#[test] +fn tuplet_with_a_zero_term_is_read_unscaled() { + let json = |s| serde_json::to_string(&read_everywhere(s)).unwrap(); + assert_eq!(json(r"{ \tuplet 0/2 { c8 d e } }"), json(r"{ c8 d e }")); + assert_eq!(json(r"{ \times 2/0 { c8 d e } }"), json(r"{ c8 d e }")); +} + +#[test] +fn figured_bass_with_hundreds_of_dots_does_not_overflow() { + // The dot count was a `u8` incremented without saturation. + let src = format!(r"figs = \figuremode {{ <6>4{} }} {{ c4 }}", ".".repeat(300)); + let _ = read_everywhere(&src); + let src = format!(r"\figures {{ <6>4{} }}", ".".repeat(300)); + let _ = read_everywhere(&src); +} + +// --- Inputs the 0.3.0 panic hunt reproduced ------------------------------------- +// +// Each was a panic, a hang, a stack overflow or an out-of-memory abort in lytk +// 0.2.0. Readings too large for the reader's bounds are refused (`Err`); the +// rest are read. + +fn refused(src: &str) -> bool { + LyToIrAdapter::new().convert_str(src).is_err() +} + +fn note_count(score: &_core::ir::score::Score) -> usize { + use _core::ir::note::VoiceElement; + score + .parts() + .iter() + .flat_map(|p| &p.measures) + .flat_map(|m| &m.voices) + .flat_map(|v| &v.elements) + .map(|e| match e { + VoiceElement::Note(_) => 1, + VoiceElement::Chord(c) => c.notes.len(), + VoiceElement::Rest(_) => 0, + }) + .sum() +} + +#[test] +fn oversized_music_is_refused_quickly() { + let start = std::time::Instant::now(); + for src in [ + r"{ c'1*1000000000 }", + r"{ c'4*9223372036854775807 c'4 }", + r"{ s1*1000000000 c'4 }", + r"{ R1*1000000000 }", + r"{ \skip 1*1000000000 c'4 }", + r"{ \set Score.measureLength = #(ly:make-moment 1 4294967295) c'1 c'1 }", + r"{ \repeat unfold 255 { \repeat unfold 255 { \repeat unfold 255 { c'4 } } } }", + r"{ \repeat unfold 99999999999999 { } }", + ] { + assert!(refused(src), "{src}"); + } + // Twenty-two definitions, each doubling the one before, never used. + let mut src = String::from("varA = { c'4 }\n"); + let names: Vec = (0..22) + .map(|i| format!("var{}", (b'A' + i as u8) as char)) + .collect(); + for w in names.windows(2) { + src.push_str(&format!("{} = {{ \\{} \\{} }}\n", w[1], w[0], w[0])); + } + assert!(refused(&src), "doubling variables"); + assert!(start.elapsed().as_secs() < 30, "took {:?}", start.elapsed()); +} + +#[test] +fn coprime_durations_are_refused_not_overflowed() { + assert!(refused( + r"{ c'4*1/251 c'4*1/241 c'4*1/239 c'4*1/233 c'4*1/229 c'4*1/227 c'4*1/223 c'4*1/211 }" + )); + let tuplets: String = [3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53] + .iter() + .map(|p| format!(r"\tuplet {p}/1 {{ c'4 }} ")) + .collect(); + assert!(refused(&format!("{{ {tuplets} }}"))); +} + +#[test] +fn a_self_referencing_variable_inside_relative_is_read_once() { + // `a = { \a \a \a }` refers to the previous `a`: three notes, not 3^16. + let score = LyToIrAdapter::new() + .convert_str("a = { c4 }\na = { \\a \\a \\a }\n\\relative c' { \\a }") + .expect("reads"); + assert_eq!(note_count(&score), 3); +} + +#[test] +fn deep_nesting_below_the_bound_does_not_overflow_the_stack() { + for (open, n) in [ + (r"\tuplet 3/2 { ", 400), + (r"\relative c' { ", 400), + (r"\repeat volta 2 { ", 400), + (r"\times 2/3 { ", 400), + ] { + let src = format!("{{ {}c'4 {} }}", open.repeat(n), "} ".repeat(n)); + let _ = LyToIrAdapter::new().convert_str(&src); + } + // Variables read again inside \relative multiply the walk's depth. + let src = format!( + "a = {{ c'4 }}\na = {{ {}\\a {}}}\n\\relative c' {{ \\a }}", + r"\relative c' { ".repeat(200), + "} ".repeat(200) + ); + let _ = LyToIrAdapter::new().convert_str(&src); +} + +#[test] +fn staff_groups_beyond_u8_are_refused_and_wide_scores_lower() { + let group = |n: usize| format!(r"\new PianoStaff << {}>>", r"\new Staff { c'4 } ".repeat(n)); + assert!(refused(&group(256))); + assert!(!refused(&group(255))); + let wide = format!("<< {}>>", r"\new Staff { c'4 } ".repeat(256)); + let doc = LyToIrAdapter::new() + .convert_str_to_music(&wide) + .expect("reads"); + let _ = _core::ir::lower::lower_to_score(&doc); +} + +#[test] +fn a_climbing_relative_passage_is_refused() { + let src = format!(r"\relative c {{ c{} c c c }}", "'".repeat(10_000)); + assert!(refused(&src)); +} + +#[test] +fn writers_handle_zero_length_notes_and_coprime_tuplets() { + let score = LyToIrAdapter::new() + .convert_str(r"{ c'4*0 d'4 }") + .expect("reads"); + assert!(IrToLyAdapter::new().convert(&score).unwrap().contains("*0")); + let voices: Vec = [251, 241, 239, 233, 229, 227, 223, 211] + .iter() + .map(|p| format!(r"{{ \tuplet {p}/1 {{ c'4 }} }}")) + .collect(); + let score = LyToIrAdapter::new() + .convert_str(&format!(r"<< {} >>", voices.join(r" \\ "))) + .expect("reads"); + let _ = IrToMxmlAdapter::new() + .convert(&score) + .expect("writes MusicXML"); +} + +#[test] +fn figured_bass_at_a_grace_note_is_written() { + // A grace note takes no time: the MusicXML writer asked its figures for + // an empty range past the one it had written, and panicked. + let score = LyToIrAdapter::new() + .convert_str( + r"<< \new Staff { \grace c'8 d'4 e'2. } \new FiguredBass \figuremode { <6>4 <5> } >>", + ) + .expect("reads"); + let xml = IrToMxmlAdapter::new() + .convert(&score) + .expect("writes MusicXML"); + assert_eq!(xml.matches("\ + P\ + 1\ + C4\ + 400000000"; + let read = MxmlToIrAdapter::new().convert_str(xml); + assert!(read.unwrap_err().to_string().contains(long)); + let kern = format!("**kern\n{}*-\n", "0c\n".repeat(60_000)); + let read = HumdrumToIrAdapter::new().convert_str(&kern); + assert!(read.unwrap_err().to_string().contains(long)); +} + +#[test] +fn the_abc_writer_refuses_hand_made_music_of_millions_of_bars() { + use _core::ir::duration::{Duration, Frac}; + use _core::ir::music::{Music, MusicDocument}; + use _core::ir::pitch::{Pitch, PitchStep}; + let note = Music::Note { + pitch: Pitch::new(PitchStep::C, 4), + duration: Duration::new(Frac::from_integer(1 << 30)), + annotations: Vec::new(), + }; + let doc = MusicDocument::new(Music::Sequential(vec![note])); + let start = std::time::Instant::now(); + assert!(IrToAbcAdapter::new().convert_music(&doc).is_err()); + assert!(start.elapsed().as_secs() < 30, "took {:?}", start.elapsed()); +} + +/// Seconds `f` takes on `n` and on `4 * n` repetitions. +fn times(n: usize, f: impl Fn(usize)) -> (f64, f64) { + let time = |k: usize| { + let start = std::time::Instant::now(); + f(k); + start.elapsed().as_secs_f64() + }; + (time(n), time(4 * n)) +} + +#[test] +fn repeated_constructs_scale_linearly() { + // Found by J1's hunt as quadratic: meter changes (`Grid::build` scanned + // every meter at each), runs of tuplets in the ABC writer (each group + // counted the run to its end), runs of grace notes in the Music-path + // LilyPond writer (each copied the group so far), diagnostics on one long + // line (each counted its column from the line's start). Four times the + // input must take about four times as long, not sixteen. + let meters = |n: usize| { + let src = format!( + "{{ {} }}", + r"\time 3/8 c'8 d' e' \time 2/4 c'4 d' ".repeat(n) + ); + let score = LyToIrAdapter::new().convert_str(&src).unwrap(); + let _ = _core::ir::lower::lower_to_score(&_core::ir::lift::lift_to_music(&score)); + }; + let tuplets = |n: usize| { + let src = format!("{{ {} }}", r"\tuplet 3/2 { c'8 d' e' } ".repeat(n)); + let score = LyToIrAdapter::new().convert_str(&src).unwrap(); + let doc = _core::ir::lift::lift_to_music(&score); + IrToAbcAdapter::new().convert_music(&doc).unwrap(); + }; + let graces = |n: usize| { + let grace = "C4eighth"; + let xml = format!( + "\ + P\ + 1\ + {}C4\ + 1", + grace.repeat(n) + ); + let score = MxmlToIrAdapter::new().convert_str(&xml).unwrap(); + let doc = _core::ir::lift::lift_to_music(&score); + IrToLyAdapter::new().convert_music(&doc).unwrap(); + }; + let diagnostics = |n: usize| { + let src = format!("{{ {} }}", r"c'3 \foo ".repeat(n)); + assert_eq!(check(&src, true).len(), 2 * n); + }; + for (name, (small, big)) in [ + ("diagnostics on one line", times(2000, diagnostics)), + ("meter changes", times(1000, meters)), + ("tuplet runs", times(2000, tuplets)), + ("grace runs", times(2000, graces)), + ] { + assert!( + big < 8.0 * small + 0.05, + "{name}: {small:.3} s, then {big:.3} s" + ); + } +} diff --git a/tests/ly_corpus.rs b/tests/ly_corpus.rs new file mode 100644 index 0000000..b5598f5 --- /dev/null +++ b/tests/ly_corpus.rs @@ -0,0 +1,435 @@ +//! LilyPond's diagnostics against corpora: how often valid LilyPond is +//! reported as wrong, and how often broken LilyPond is caught (Epic J). +//! +//! - `mutation_board` breaks the committed `.ly` fixtures one token at a time +//! (a brace, a quote, a `<<`, a `>>`, a chord's `>`, a Scheme parenthesis) +//! and counts the broken files with an error: from the syntax check alone +//! (`check(text, false)`) and from a reading (`check(text, true)`). It runs +//! in every `cargo test`. +//! - `syntax_board` checks the syntax of LilyPond's own regression tests and +//! documentation snippets, all valid LilyPond, and counts the files with an +//! error; `reader_board` reads them all, counts the files with an error and +//! the ones refused as too large (its bounds must not catch real music), and +//! tallies the warnings per code. Both need LilyPond's sources, so they are +//! ignored by default: +//! `LYTK_LILYPOND_SRC=/path/to/lilypond cargo test --test ly_corpus -- --ignored --nocapture` +//! (`LYTK_LILYPOND_SRC` defaults to the `lilypond/` reference checkout; CI +//! sparse-clones v2.26.0). + +use _core::adapters::ly_to_ir::{check, header_fields, LyToIrAdapter}; +use _core::parser::LilyPondParser; +use tree_sitter::Tree; + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +// Files of LilyPond v2.26.0's `input/regression` (recursive) and +// `Documentation/snippets` with a syntax error (2026-09-27): +// `other/display-lily-tests.ly` (`##[ #]`, which the grammar does not know). +// It may only fall. +const SYNTAX_ERROR_FILES: usize = 1; +/// Valid files the reader reports an error in (syntax or semantic): the same +/// one. It may only fall. +const READER_ERROR_FILES: usize = 1; +/// Files named per warning code in the reader board's output. +const WARNING_SAMPLES: usize = 5; +/// The corpus must be at least this large, so a wrong path fails loudly +/// instead of passing on an empty directory. +const SYNTAX_MIN_FILES: usize = 2500; + +// Broken fixtures caught per kind of deleted token, by the syntax check and by +// a reading, out of the mutations made (2026-09-27: 100, 100, 46, 46, 21, 105, +// 42, 42). They may only rise. The three Scheme `(` missed leave valid +// LilyPond: `#'(4 4)` without its `(` is `#'4` and the note `4)`. +const MUTATIONS_CAUGHT: &[(&str, usize, usize)] = &[ + ("open brace `{`", 100, 100), + ("close brace `}`", 100, 100), + ("open `<<`", 46, 46), + ("close `>>`", 46, 46), + ("chord `>`", 21, 21), + ("string quote", 105, 105), + ("Scheme `(`", 39, 39), + ("Scheme `)`", 42, 42), +]; + +fn read_lossy(path: &Path) -> String { + let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + String::from_utf8_lossy(&bytes).into_owned() +} + +fn ly_files(dir: &Path, recursive: bool) -> Vec { + let mut out = Vec::new(); + let Ok(entries) = std::fs::read_dir(dir) else { + return out; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if recursive { + out.extend(ly_files(&path, true)); + } + } else if path.extension().is_some_and(|e| e == "ly") { + out.push(path); + } + } + out.sort(); + out +} + +fn corpus_root() -> Option { + let root = std::env::var_os("LYTK_LILYPOND_SRC") + .map(PathBuf::from) + .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("lilypond")); + root.join("input/regression").is_dir().then_some(root) +} + +#[test] +#[ignore = "needs LilyPond's sources (set LYTK_LILYPOND_SRC)"] +fn syntax_board() { + let Some(root) = corpus_root() else { + eprintln!("syntax_board skipped: no LilyPond sources (set LYTK_LILYPOND_SRC)"); + return; + }; + let regression = root.join("input/regression"); + let sets = [ + ("input/regression/*.ly", ly_files(®ression, false)), + ( + "input/regression/*/**.ly", + ly_files(®ression, true) + .into_iter() + .filter(|p| p.parent() != Some(regression.as_path())) + .collect(), + ), + ( + "Documentation/snippets/*.ly", + ly_files(&root.join("Documentation/snippets"), false), + ), + ]; + + let (mut total, mut flagged_files) = (0, Vec::new()); + let mut times = Vec::new(); + for (name, files) in &sets { + let start = Instant::now(); + let mut bad = 0; + for path in files { + let src = read_lossy(path); + let t = Instant::now(); + let errors: Vec = check(&src, false) + .iter() + .filter(|d| d.is_error()) + .map(|d| d.to_string()) + .collect(); + times.push(t.elapsed()); + if !errors.is_empty() { + bad += 1; + let rel = path.strip_prefix(&root).unwrap_or(path); + flagged_files.push(format!( + "{}: {:?}", + rel.display(), + &errors[..errors.len().min(3)] + )); + } + } + let ms = start.elapsed().as_secs_f64() * 1e3; + println!( + "{name:28} {:5} files {bad:3} with errors {:.3} ms/file", + files.len(), + ms / files.len().max(1) as f64 + ); + total += files.len(); + } + times.sort(); + let median = times.get(times.len() / 2).copied().unwrap_or_default(); + println!( + "check_lilypond median: {:.3} ms", + median.as_secs_f64() * 1e3 + ); + for f in &flagged_files { + println!(" error: {f}"); + } + assert!( + total >= SYNTAX_MIN_FILES, + "only {total} files under {}: wrong LYTK_LILYPOND_SRC?", + root.display() + ); + assert!( + flagged_files.len() <= SYNTAX_ERROR_FILES, + "{} valid files with syntax errors, baseline {SYNTAX_ERROR_FILES}", + flagged_files.len() + ); + if flagged_files.len() < SYNTAX_ERROR_FILES { + println!( + "SYNTAX_ERROR_FILES can be lowered to {}", + flagged_files.len() + ); + } +} + +/// Every valid file of the corpus, for the boards that read them all. +fn corpus_files(root: &Path) -> Vec { + let mut files = ly_files(&root.join("input/regression"), true); + files.extend(ly_files(&root.join("Documentation/snippets"), false)); + files +} + +#[test] +#[ignore = "needs LilyPond's sources (set LYTK_LILYPOND_SRC)"] +fn reader_board() { + let Some(root) = corpus_root() else { + eprintln!("reader_board skipped: no LilyPond sources (set LYTK_LILYPOND_SRC)"); + return; + }; + let files = corpus_files(&root); + assert!(files.len() >= SYNTAX_MIN_FILES, "wrong LYTK_LILYPOND_SRC?"); + let reader = LyToIrAdapter::new(); + let start = Instant::now(); + let (mut other_errors, mut refused, mut with_errors) = (0, Vec::new(), Vec::new()); + // Board (d): warnings per code, as (count, files). + let mut warnings: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + let mut samples: BTreeMap<&str, Vec> = BTreeMap::new(); + for path in &files { + let rel = path + .strip_prefix(&root) + .unwrap_or(path) + .display() + .to_string(); + match reader.read_str(&read_lossy(path)) { + Ok(reading) => { + let errors: Vec<_> = reading + .diagnostics + .iter() + .filter(|d| d.is_error()) + .collect(); + if !errors.is_empty() { + with_errors.push(format!("{rel}: {}", errors[0])); + } + let mut seen = std::collections::BTreeSet::new(); + for d in reading.diagnostics.iter().filter(|d| !d.is_error()) { + let entry = warnings.entry(d.code).or_default(); + entry.0 += 1; + if seen.insert(d.code) { + entry.1 += 1; + let list = samples.entry(d.code).or_default(); + if list.len() < WARNING_SAMPLES { + list.push(format!("{rel}: {d}")); + } + } + } + } + Err(e) if e.to_string().contains("refuses") => { + refused.push(format!("{rel}: {e}")); + } + Err(_) => other_errors += 1, + } + } + println!( + "{} files read in {:.1} s: {} refused, {other_errors} other errors, {} with errors", + files.len(), + start.elapsed().as_secs_f64(), + refused.len(), + with_errors.len() + ); + for r in &refused { + println!(" refused: {r}"); + } + for e in &with_errors { + println!(" error: {e}"); + } + println!("warnings (count, files):"); + for (code, (count, in_files)) in &warnings { + println!(" {code:20} {count:6} {in_files:5}"); + for sample in &samples[code] { + println!(" {sample}"); + } + } + // The reader's bounds are orders of magnitude above real music: no valid + // file may reach one. + assert!(refused.is_empty(), "{} valid files refused", refused.len()); + assert!( + with_errors.len() <= READER_ERROR_FILES, + "{} valid files with errors, baseline {READER_ERROR_FILES}", + with_errors.len() + ); +} + +/// `key = "…"` of `src`, found by text search alone and decoded with +/// LilyPond's escapes (`lily/lexer.ll`): an oracle independent of the grammar. +fn decode_field(src: &str, key: &str) -> Option { + let at = src.find(&format!("{key} = \""))? + key.len() + 4; + let mut out = String::new(); + let mut chars = src[at..].chars(); + while let Some(c) = chars.next() { + match c { + '"' => return Some(out), + '\\' => match chars.next()? { + 'n' => out.push('\n'), + 't' => out.push('\t'), + e @ ('\\' | '"' | '\'') => out.push(e), + e => { + out.push('\\'); + out.push(e); + } + }, + c => out.push(c), + } + } + None +} + +#[test] +#[ignore = "needs LilyPond's sources (set LYTK_LILYPOND_SRC)"] +fn snippet_headers() { + let Some(root) = corpus_root() else { + eprintln!("snippet_headers skipped: no LilyPond sources (set LYTK_LILYPOND_SRC)"); + return; + }; + let files = ly_files(&root.join("Documentation/snippets"), false); + assert!(files.len() >= 380, "wrong LYTK_LILYPOND_SRC?"); + let (mut compared, mut differ) = (0, Vec::new()); + for path in &files { + let src = read_lossy(path); + let fields = header_fields(&src); + for key in ["texidoc", "categories"] { + let expected = decode_field(&src, key); + let found = fields + .iter() + .find(|f| f.key == key && f.score.is_none()) + .map(|f| f.value.clone()); + compared += usize::from(expected.is_some()); + if found != expected { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + differ.push(format!("{name} {key}: {found:?} != {expected:?}")); + } + } + } + println!( + "{compared} texidoc/categories values of {} snippets compared", + files.len() + ); + for d in &differ { + println!(" differs: {d}"); + } + assert!(differ.is_empty(), "{} fields differ", differ.len()); +} + +/// The token kinds a mutation deletes, with their board label. A token is a +/// leaf of the tree, so a `{` inside a string or a comment is never picked. +/// A slur's `(` or `)` is not one: without it, the music is still valid. +const MUTATION_KINDS: &[(&str, &str, Option<&str>)] = &[ + // (label, token kind, required parent kind) + ("open brace `{`", "{", None), + ("close brace `}`", "}", None), + ("open `<<`", "<<", None), + ("close `>>`", ">>", None), + ("chord `>`", ">", Some("chord")), + ("string quote", "\"", None), + ("Scheme `(`", "(", Some("scheme_list")), + ("Scheme `)`", ")", Some("scheme_list")), +]; + +fn token_ranges(tree: &Tree, kind: &str, parent: Option<&str>) -> Vec<(usize, usize)> { + let mut out = Vec::new(); + let mut stack = vec![tree.root_node()]; + while let Some(node) = stack.pop() { + if node.child_count() == 0 { + if !node.is_named() + && node.kind() == kind + && parent.is_none_or(|p| node.parent().is_some_and(|n| n.kind() == p)) + { + out.push((node.start_byte(), node.end_byte())); + } + continue; + } + let mut cursor = node.walk(); + stack.extend(node.children(&mut cursor)); + } + out.sort(); + out +} + +/// Up to three occurrences per file and kind: the first, the middle and the last. +fn picks(n: usize) -> Vec { + let mut v: Vec = [0, n / 2, n.saturating_sub(1)] + .into_iter() + .filter(|&i| i < n) + .collect(); + v.dedup(); + v +} + +fn has_error(text: &str, semantic: bool) -> bool { + check(text, semantic).iter().any(|d| d.is_error()) +} + +#[test] +fn mutation_board() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ly"); + let files = ly_files(&dir, false); + assert!( + files.len() >= 30, + "fixtures missing under {}", + dir.display() + ); + + let mut parser = LilyPondParser::new().expect("grammar loads"); + // label → (caught by the syntax check, caught by the reading, made) + let mut board: BTreeMap<&str, (usize, usize, usize)> = BTreeMap::new(); + let mut missed = Vec::new(); + for path in &files { + let src = read_lossy(path); + let tree = parser.parse(&src).expect("fixture parses"); + let before = check(&src, true); + assert!( + !before.iter().any(|d| d.is_error()), + "{} has an error before any mutation: {}", + path.display(), + before.iter().find(|d| d.is_error()).unwrap() + ); + for &(label, kind, parent) in MUTATION_KINDS { + let ranges = token_ranges(&tree, kind, parent); + for i in picks(ranges.len()) { + let (a, b) = ranges[i]; + let broken = format!("{}{}", &src[..a], &src[b..]); + // The reading reports every syntax error too. + let by_syntax = has_error(&broken, false); + let by_reading = by_syntax || has_error(&broken, true); + let entry = board.entry(label).or_default(); + entry.0 += usize::from(by_syntax); + entry.1 += usize::from(by_reading); + entry.2 += 1; + if !by_reading { + let line = src[..a].matches('\n').count() + 1; + missed.push(format!( + "{} deleted at {}:{line}", + label, + path.file_name().unwrap().to_string_lossy() + )); + } + } + } + } + + println!( + "{:18} {:>6} {:>7} {:>5}", + "deleted", "syntax", "reading", "made" + ); + for (label, (syntax, reading, made)) in &board { + println!("{label:18} {syntax:6} {reading:7} {made:5}"); + } + for m in &missed { + println!(" not caught: {m}"); + } + for &(label, syntax_baseline, reading_baseline) in MUTATIONS_CAUGHT { + let (syntax, reading, made) = board.get(label).copied().unwrap_or_default(); + assert!( + made > 0, + "no `{label}` token in the fixtures: grammar changed?" + ); + assert!( + syntax >= syntax_baseline && reading >= reading_baseline, + "{label}: {syntax} (syntax) and {reading} (reading) of {made} caught, \ + baselines {syntax_baseline} and {reading_baseline}" + ); + } +} diff --git a/tests/ly_diagnostics.rs b/tests/ly_diagnostics.rs new file mode 100644 index 0000000..f8c07d5 --- /dev/null +++ b/tests/ly_diagnostics.rs @@ -0,0 +1,167 @@ +//! Diagnostics of the LilyPond reader (Epic J3): each code on the input that +//! raises it, and no false alarm on the constructs it must leave alone. + +use _core::adapters::ly_to_ir::{check, read_source, LyToIrAdapter}; +use _core::adapters::AdapterError; +use _core::diagnostics::{Diagnostic, Severity}; + +fn codes(text: &str) -> Vec<(Severity, &'static str)> { + check(text, true) + .iter() + .map(|d| (d.severity, d.code)) + .collect() +} + +#[test] +fn each_code_on_its_input() { + use Severity::{Error, Warning}; + let cases: &[(&str, Severity, &str)] = &[ + // Syntax, from the tree. + ("{ c'4 d'4", Error, "missing-token"), + ("\\score { \\new Staff { c'1 } >> }", Error, "syntax-error"), + ("#set-global-staff-size 20)\n{ c'1 }", Error, "syntax-error"), + ( + "{ \\override Beam.positions = #'1 . 2) c'1 }", + Error, + "syntax-error", + ), + ( + "\\markup { \\override #'box-padding \"x\" }", + Error, + "syntax-error", + ), + // Semantic errors, from the walk. + ("{ c'3 }", Error, "invalid-duration"), + ("{ \\tuplet 0/2 { c'8 d' e' } }", Error, "invalid-ratio"), + ("{ c'4*1/0 }", Error, "invalid-ratio"), + ("Hello world, this is not music.", Error, "not-lilypond"), + // Warnings: what the reader does not read. + ("{ c'2048 }", Warning, "unsupported-value"), + ("{ \\time 3/256 c'4 }", Warning, "unsupported-value"), + ("{ \\time 1/0 c'1 }", Warning, "unsupported-value"), + ("{ \\tuplet 300/2 { c'8 } }", Warning, "unsupported-value"), + ( + "\\include \"notes.ily\"\n{ c'1 }", + Warning, + "ignored-include", + ), + ("{ c'4 \\noSuchCommand d'4 }", Warning, "unknown-command"), + ( + "\\language \"klingon\"\n{ c'1 }", + Warning, + "unknown-language", + ), + ( + "\\score { { c'1 } \\midi { } }\n\\score { { d'1 } \\layout { } }", + Warning, + "skipped-score", + ), + ("c'4 d' e'", Error, "syntax-error"), + ("\\drums { bd4 xyz }", Warning, "unrecognized-token"), + ]; + for &(text, severity, code) in cases { + let found = codes(text); + assert!( + found.contains(&(severity, code)), + "{text:?}: expected {} [{code}], got {found:?}", + severity.as_str() + ); + } +} + +#[test] +fn valid_lilypond_raises_nothing() { + let cases = [ + "\\version \"2.24.0\"\n\\header { title = \"T\" }\n\\relative c' { c4( d) e-> f\\p }", + // A post-event function's parenthesis, and a tweak's. + "{ c1 \\vshape #'((0 . 0) (0 . 1)) ( d1) e-\\tweak color #red ( f) }", + // Markup is text: marks and words after Scheme values are fine. + "\\markup { \\hspace #-1.0 . \\teeny .org }", + // Commands the file defines, in Scheme or as quoted identifiers. + "#(define (twice m) m)\nfoo = #(define-music-function (m) (ly:music?) m)\n\ + \"\\\\|\" = \\bar \"|\"\n{ \\foo c'1 \\| }", + // LilyPond reads a byte-order mark anywhere as whitespace. + "{ c'4 \u{feff}d'4 }", + "violin.1 = { c'1 }", + "\\layout { \\context { \\Score \\override SpacingSpanner.x = #1 } }\n{ c'1 }", + ]; + for text in cases { + let found = check(text, true); + assert!(found.is_empty(), "{text:?}: {found:?}"); + } +} + +#[test] +fn markup_words_are_text_not_notes() { + // LilyPond reads `a8` here as the markup's word: its bar check fails. + let reader = LyToIrAdapter::new(); + let reading = reader + .read_str("{ \\time 2/4 c'4 -\\markup \\bold a8 d'4 \\markup { e f } }") + .unwrap(); + let notes: usize = reading.scores[0].parts()[0] + .measures + .iter() + .flat_map(|m| &m.voices) + .map(|v| v.elements.len()) + .sum(); + assert_eq!(notes, 2, "{:?}", reading.scores[0]); +} + +#[test] +fn a_single_score_reading_warns_about_the_movements_it_drops() { + let reading = LyToIrAdapter::new() + .read_str("\\score { { c'1 } }\n\\score { { d'1 } }\n\\score { { e'1 } }") + .unwrap(); + assert_eq!(reading.scores.len(), 2 + 1); + assert!(reading.diagnostics.is_empty(), "{:?}", reading.diagnostics); + let (_, diagnostics) = reading.into_first(); + let lines: Vec = diagnostics + .iter() + .filter(|d| d.code == "dropped-music") + .map(|d| d.line) + .collect(); + assert_eq!(lines, [2, 3]); +} + +#[test] +fn position_and_display() { + // The column counts characters, not bytes; the range is in bytes. + let text = "{ é c'4 >> }"; + let d = check(text, false).remove(0); + assert_eq!((d.line, d.column), (1, 9)); + assert_eq!(&text[d.start..d.end], ">>"); + assert_eq!( + d.to_string(), + "1:9: error: `>>` without a matching `<<` [syntax-error]" + ); + // A brace left open at the end is reported at the brace. + let d = check("\\version \"2.24.0\"\n{ c'4 d'4", false).remove(0); + assert_eq!((d.code, d.line, d.column), ("missing-token", 2, 1)); + assert!(d.message.starts_with("missing `}`"), "{d}"); +} + +#[test] +fn input_past_the_bounds_is_a_diagnostic_when_checked() { + let deep = format!("{}{}", "{ ".repeat(3000), " }".repeat(3000)); + for semantic in [false, true] { + let found = check(&deep, semantic); + assert!( + found + .iter() + .any(|d: &Diagnostic| d.code == "too-large" && d.is_error()), + "{found:?}" + ); + } +} + +#[test] +fn a_file_that_is_not_utf8_is_a_parse_error() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("latin1.ly"); + std::fs::write(&path, b"{ c'4 \xe9 }").unwrap(); + assert!(matches!(read_source(&path), Err(AdapterError::Parse(_)))); + assert!(matches!( + read_source(&dir.path().join("missing.ly")), + Err(AdapterError::Io(_)) + )); +} diff --git a/tests/ly_modes.rs b/tests/ly_modes.rs new file mode 100644 index 0000000..523b9c5 --- /dev/null +++ b/tests/ly_modes.rs @@ -0,0 +1,124 @@ +//! Input modes and movements of the LilyPond reader: `\drums`, `\chords`, +//! `\figures`, `\lyrics`, `\fixed`, and a movement per top-level music +//! expression, as LilyPond 2.22 reads them. + +use _core::adapters::ly_to_ir::{check, LyToIrAdapter}; +use _core::ir::measure::ClefSign; +use _core::ir::note::VoiceElement; +use _core::ir::Score; + +fn read(text: &str) -> Vec { + LyToIrAdapter::new().read_str(text).expect("reads").scores +} + +/// MIDI numbers of each part's notes, chords flattened. +fn midi(score: &Score) -> Vec> { + score + .parts() + .iter() + .map(|p| { + p.measures + .iter() + .flat_map(|m| &m.voices) + .flat_map(|v| &v.elements) + .flat_map(|e| match e { + VoiceElement::Note(n) => vec![n.pitch.midi_number()], + VoiceElement::Chord(c) => { + c.notes.iter().map(|n| n.pitch.midi_number()).collect() + } + _ => vec![], + }) + .collect() + }) + .collect() +} + +fn codes(text: &str) -> Vec<&'static str> { + check(text, true).iter().map(|d| d.code).collect() +} + +#[test] +fn drum_names_are_general_midi_keys_on_a_percussion_staff() { + let src = "\\drums { bd4 sn hhc }"; + let score = &read(src)[0]; + assert_eq!(midi(score), [vec![36, 38, 36, 42, 42]]); + let clefs = &score.parts()[0].measures[0] + .attributes + .as_ref() + .unwrap() + .clefs; + assert_eq!(clefs[&1].sign, ClefSign::Percussion); + assert_eq!(codes(src), Vec::<&str>::new()); + // A drum voice in a drum staff, and a drum-mode variable. + let src = "d = \\drummode { bd4 sn }\n\\new DrumStaff \\new DrumVoice \\drummode { \\d tomh }"; + assert_eq!(midi(&read(src)[0]), [vec![36, 38, 50]]); + assert_eq!(codes("\\drums { bd4 xyz }"), ["unrecognized-token"]); +} + +#[test] +fn chord_figure_and_lyric_shorthands_make_no_notes() { + let src = "<< \\chords { c2 g:7 } \\figures { <6>2 <6 4> } \\new Staff { e'2 d' } \ + \\lyrics { la la } >>"; + let score = &read(src)[0]; + assert_eq!(midi(score), [vec![64, 62]]); + let m = &score.parts()[0].measures[0]; + assert_eq!(m.harmonies.len(), 2); + assert_eq!(m.figured_bass.len(), 2); + assert_eq!(codes(src), Vec::<&str>::new()); +} + +#[test] +fn fixed_pitches_are_absolute_an_octave_up_per_mark() { + // c' is MIDI 60; `\fixed c'` puts unmarked notes in its octave. + assert_eq!( + midi(&read("\\fixed c' { c4 g b, d'' }")[0]), + [vec![60, 67, 59, 86]] + ); + // Inside `\relative`, a `\fixed` block is absolute, and after it the + // relative reading goes on from before it. + assert_eq!( + midi(&read("\\relative c'' { c4 \\fixed c { c e } d }")[0]), + [vec![72, 48, 52, 74]] + ); + assert_eq!( + midi(&read("m = \\fixed c'' { c4 d }\n\\score { \\m }")[0]), + [vec![72, 74]] + ); +} + +#[test] +fn each_top_level_music_expression_is_a_movement() { + let first = |scores: &[Score]| scores.iter().map(|s| midi(s)[0][0]).collect::>(); + // LilyPond makes a score of each, in order, beside `\score` blocks. + let src = "{ c'1 }\n\\score { { d'1 } }\n\\new Staff { e'1 }\nm = { f'1 }\n\\m"; + assert_eq!(first(&read(src)), [60, 62, 64, 65]); + // `\addlyrics` belongs to the expression before it. + let scores = read("{ c'4 d' } \\addlyrics { la la }\n{ e'1 }"); + assert_eq!(scores.len(), 2); + let lyrics: usize = scores[0] + .parts() + .iter() + .flat_map(|p| &p.measures) + .flat_map(|m| &m.voices) + .flat_map(|v| &v.elements) + .map(|e| match e { + VoiceElement::Note(n) => n.lyrics.len(), + _ => 0, + }) + .sum(); + assert_eq!(lyrics, 2); + // A book's scores and music are movements too, and its header the book's. + let src = "\\book { \\header { title = \"B\" } \\score { { c'1 } } { d'1 } }"; + let scores = read(src); + assert_eq!(first(&scores), [60, 62]); + assert!(scores + .iter() + .all(|s| s.metadata.title.as_deref() == Some("B"))); + assert_eq!(codes(src), Vec::<&str>::new()); +} + +#[test] +fn notes_outside_braces_are_a_syntax_error() { + // LilyPond: "syntax error, unexpected NOTENAME_PITCH". + assert_eq!(codes("\\version \"2.24.0\"\nc'4 d' e'"), ["syntax-error"]); +} diff --git a/tests/ly_text.rs b/tests/ly_text.rs new file mode 100644 index 0000000..29aaaa8 --- /dev/null +++ b/tests/ly_text.rs @@ -0,0 +1,255 @@ +//! LilyPond strings and headers (Epic J4), and pitch languages (J5). + +use _core::adapters::ir_to_ly::IrToLyAdapter; +use _core::adapters::ly_to_ir::{check, header_fields, LyToIrAdapter}; +use _core::adapters::{FromIrAdapter, FromMusicAdapter, ToIrAdapter}; +use _core::ir::lift::lift_to_music; +use _core::ir::note::VoiceElement; +use _core::ir::pitch::PitchStep; +use _core::ir::Score; + +/// Strings to decode: a quote, a backslash, a line break, non-ASCII. +const TRICKY: [&str; 4] = ["say \"hi\"", r"a\b", "one\ntwo", "Émile — 漢字 ü"]; + +/// `s` as a LilyPond string literal. +fn ly(s: &str) -> String { + let escaped = s + .replace('\\', r"\\") + .replace('"', "\\\"") + .replace('\n', r"\n"); + format!("\"{escaped}\"") +} + +fn read(text: &str) -> Score { + LyToIrAdapter::new().convert_str(text).expect("reads") +} + +/// Every text of `score` that a string of the source reaches. +fn texts(score: &Score) -> Vec { + let meta = &score.metadata; + let mut out: Vec = [&meta.title, &meta.composer, &meta.lyricist] + .into_iter() + .flatten() + .cloned() + .chain(meta.extra.values().cloned()) + .collect(); + for part in score.parts() { + out.push(part.name.clone()); + for m in &part.measures { + for d in &m.directions { + out.extend(d.tempo.as_ref().and_then(|t| t.text.clone())); + out.extend(d.da_capo.clone()); + } + for v in &m.voices { + for e in &v.elements { + if let VoiceElement::Note(n) = e { + out.extend(n.text_directions.iter().map(|t| t.text.clone())); + out.extend(n.lyrics.iter().map(|l| l.text.clone())); + } + } + } + } + } + out.retain(|t| !t.is_empty()); + out.sort(); + out +} + +#[test] +fn strings_decode_and_round_trip_in_every_context() { + let [q, b, n, u] = TRICKY.map(ly); + let dc = ly(&format!("D.C. {}", TRICKY[0])); + let src = format!( + "\\header {{ title = {q} composer = {b} opus = {n} copyright = {u} }}\n\ + \\new Staff \\with {{ instrumentName = {u} }} {{\n\ + \\tempo {q} 4 = 60\n\ + c'4^\\markup {{ {b} }} d'4 \\mark {dc} e'4 f'4\n\ + }}\n\ + \\addlyrics {{ {n} {q} x y }}\n" + ); + let first = texts(&read(&src)); + for s in TRICKY { + assert!(first.iter().any(|t| t == s), "{s:?} not read: {first:?}"); + } + assert!(first.contains(&format!("D.C. {}", TRICKY[0])), "{first:?}"); + // Written back and read again, every text is the same. + let written = IrToLyAdapter::new().convert(&read(&src)).unwrap(); + assert_eq!(texts(&read(&written)), first, "{written}"); +} + +#[test] +fn header_values_are_text() { + let meta = read( + "\\header { composer = \\markup { \\bold \"J. S.\" Bach } poet = #\"Anon \\\"X\\\"\" \ + tagline = ##f title = \\markup \\italic Sonata }", + ) + .metadata; + assert_eq!(meta.composer.as_deref(), Some("J. S. Bach")); + assert_eq!(meta.lyricist.as_deref(), Some("Anon \"X\"")); + assert_eq!(meta.title.as_deref(), Some("Sonata")); + assert!(!meta.extra.contains_key("tagline"), "{:?}", meta.extra); +} + +#[test] +fn headers_are_scoped_to_their_movement() { + let src = "\\header { title = \"Book\" composer = \"C\" }\n\ + \\score { \\header { piece = \"I\" title = \"First\" } { c'1 } }\n\ + \\score { { d'1 } }\n\ + \\header { opus = \"5\" title = \"Book 2\" }\n"; + let movements = LyToIrAdapter::new().convert_str_multi(src).unwrap(); + let (a, b) = (&movements[0].metadata, &movements[1].metadata); + assert_eq!(a.title.as_deref(), Some("First")); + assert_eq!(a.extra.get("piece").map(String::as_str), Some("I")); + // The book's header applies to every movement, its last assignment + // winning, wherever it stands. + assert_eq!(b.title.as_deref(), Some("Book 2")); + assert_eq!(a.composer.as_deref(), Some("C")); + assert_eq!(b.composer.as_deref(), Some("C")); + assert_eq!(a.extra.get("opus").map(String::as_str), Some("5")); + assert_eq!(b.extra.get("opus").map(String::as_str), Some("5")); + assert!(!b.extra.contains_key("piece"), "{:?}", b.extra); +} + +#[test] +fn header_fields_are_located_and_can_be_cut() { + let src = "\\header { title = \"A\\\"B\" }\n\ + \\score { \\header { piece = \\markup { \\italic Largo } } { c'1 } }\n\ + \\score { \\header { \"opus\" = #\"II\" } { d'1 } }\n"; + let fields: Vec<_> = header_fields(src) + .into_iter() + .map(|f| { + ( + f.key.clone(), + f.value.clone(), + f.score, + &src[f.start..f.end], + ) + }) + .collect(); + assert_eq!( + fields, + [ + ("title".into(), "A\"B".into(), None, "title = \"A\\\"B\""), + ( + "piece".into(), + "Largo".into(), + Some(0), + "piece = \\markup { \\italic Largo }" + ), + ("opus".into(), "II".into(), Some(1), "\"opus\" = #\"II\""), + ] + ); +} + +#[test] +fn every_header_field_is_written_in_order() { + let mut score = read("{ c'1 }"); + let meta = &mut score.metadata; + meta.subtitle = Some("Sub".into()); + meta.lyricist = Some("Poet".into()); + for (k, v) in [ + ("opus", "5"), + ("copyright", "©"), + ("work-number", "3"), + ("2nd key", "x"), + ] { + meta.extra.insert(k.into(), v.into()); + } + let out = IrToLyAdapter::new().convert(&score).unwrap(); + let header: Vec<&str> = out + .lines() + .skip_while(|l| *l != "\\header {") + .take_while(|l| *l != "}") + .collect(); + assert_eq!( + header, + [ + "\\header {", + " subtitle = \"Sub\"", + " poet = \"Poet\"", + " \"2nd key\" = \"x\"", + " copyright = \"©\"", + " opus = \"5\"", + " work-number = \"3\"", + ] + ); + let again = read(&out).metadata; + assert_eq!(again.extra, score.metadata.extra); + // The Music path writes the same block. + let music = IrToLyAdapter::new() + .convert_music(&lift_to_music(&score)) + .unwrap(); + assert!(music.contains(&header.join("\n")), "{music}"); +} + +/// (step, alter, octave) of the first part's notes. +fn pitches(score: &Score) -> Vec<(PitchStep, i32, i32)> { + score.parts()[0] + .measures + .iter() + .flat_map(|m| &m.voices) + .flat_map(|v| &v.elements) + .filter_map(|e| match e { + VoiceElement::Note(n) => { + Some((n.pitch.step, n.pitch.alter.to_integer(), n.pitch.octave)) + } + _ => None, + }) + .collect() +} + +#[test] +fn language_files_set_the_language() { + use PitchStep::{C, D}; + assert_eq!( + pitches(&read("\\include \"english.ly\" { cs'4 d'4 }")), + [(C, 1, 4), (D, 0, 4)] + ); + assert_eq!( + pitches(&read("\\include \"arabic.ly\" { do'4 re'4 }")), + [(C, 0, 4), (D, 0, 4)] + ); + let codes = |t: &str| -> Vec<&str> { check(t, true).iter().map(|d| d.code).collect() }; + assert_eq!( + codes("\\include \"english.ly\" { cs'4 }"), + Vec::<&str>::new() + ); + assert_eq!( + codes("\\include \"persian.ly\" { c'4 }"), + ["unknown-language"] + ); + assert_eq!( + codes("\\include \"notes.ily\" { c'4 }"), + ["ignored-include"] + ); +} + +#[test] +fn language_changes_inside_scores_and_music() { + use PitchStep::C; + assert_eq!( + pitches(&read("\\score { \\language \"english\" { cs'4 } }")), + [(C, 1, 4)] + ); + assert_eq!( + pitches(&read("{ c'4 \\language \"english\" cs'4 }")), + [(C, 0, 4), (C, 1, 4)] + ); + // An unknown name keeps the language in force. + let src = "\\language \"english\" \\language \"klingon\" { cs'4 }"; + assert_eq!(pitches(&read(src)), [(C, 1, 4)]); + let codes: Vec<&str> = check(src, true).iter().map(|d| d.code).collect(); + assert_eq!(codes, ["unknown-language"]); +} + +#[test] +fn a_variable_read_again_keeps_the_language_of_its_definition() { + // Inside `\relative` the variable is read again, in English. + let src = + "\\language \"english\"\nm = { cs'4 }\n\\language \"deutsch\"\n\\relative c' { \\m cis }"; + let p = pitches(&read(src)); + assert_eq!( + p.iter().map(|&(s, a, _)| (s, a)).collect::>(), + [(PitchStep::C, 1); 2] + ); +} diff --git a/tests/test_bindings.py b/tests/test_bindings.py index 5c36ba9..62a2df1 100644 --- a/tests/test_bindings.py +++ b/tests/test_bindings.py @@ -311,6 +311,267 @@ def test_malformed_string_raises_valueerror_not_ioerror(self): lytk.from_abc_string("\x00\x01 not abc") + +def test_version_is_the_crates(): + # One version source: Cargo.toml, which maturin puts in the metadata too. + import re + from importlib.metadata import version + + cargo = (Path(__file__).parent.parent / "Cargo.toml").read_text() + expected = re.search(r'^version = "(.+)"$', cargo, re.M).group(1) + assert lytk.__version__ == expected == version("lytk") + + +class TestExceptionHierarchy: + """Readers raise ParseError, which is a ValueError too; I/O stays OSError.""" + + def test_classes(self): + assert issubclass(lytk.ParseError, lytk.LytkError) + assert issubclass(lytk.ParseError, ValueError) + assert issubclass(lytk.LilyPondSyntaxError, lytk.ParseError) + assert issubclass(lytk.InternalError, lytk.LytkError) + assert not issubclass(lytk.InternalError, ValueError) + assert lytk.ParseError.__module__ == "lytk" + + @pytest.mark.parametrize( + "read", + [ + lambda: lytk.from_musicxml_string("not musicxml <<<"), + lambda: lytk.from_musicxml_bytes(b"not musicxml"), + lambda: lytk.from_midi_bytes(b"not a midi file"), + lambda: lytk.from_lilypond_string("{ " * 3000 + "}" * 3000), + lambda: lytk.Score.from_json("{ not json"), + lambda: lytk.Score.from_dict({"not": "a score"}), + lambda: lytk.MusicDocument.from_json("[]"), + ], + ) + def test_malformed_input_is_a_parse_error(self, read): + with pytest.raises(lytk.ParseError): + read() + # Code written against 0.2 keeps working. + with pytest.raises(ValueError): + read() + + def test_a_missing_file_is_an_os_error(self, tmp_path): + for read in (lytk.from_lilypond, lytk.from_musicxml, lytk.from_abc, lytk.from_humdrum): + with pytest.raises(OSError): + read(str(tmp_path / "missing")) + + def test_a_ly_file_that_is_not_utf8_is_a_parse_error(self, tmp_path): + path = tmp_path / "latin1.ly" + path.write_bytes(b"{ c'4 \xe9 }") + with pytest.raises(lytk.ParseError, match="not UTF-8"): + lytk.from_lilypond(str(path)) + + def test_flatten_errors_are_parse_errors(self, tmp_path): + main = tmp_path / "main.ly" + main.write_text('\\include "absent.ily"\n') + with pytest.raises(lytk.ParseError, match="not found"): + lytk.flatten(str(main)) + + def test_pickles_with_its_diagnostics(self): + import pickle + + with pytest.raises(lytk.LilyPondSyntaxError) as info: + lytk.from_lilypond_string("{ c'3 }", strict=True) + again = pickle.loads(pickle.dumps(info.value)) + assert type(again) is lytk.LilyPondSyntaxError + assert again.diagnostics == info.value.diagnostics + + +class TestDiagnostics: + """check_lilypond, strict mode and .diagnostics (Epic J3).""" + + def test_syntax_only_by_default(self): + text = "{ c'3 \\noSuchCommand }" + assert lytk.check_lilypond(text) == [] + codes = [d.code for d in lytk.check_lilypond(text, semantic=True)] + assert codes == ["invalid-duration", "unknown-command"] + + def test_fields_and_text(self): + text = "% é\n{ é c'4 >> }" + (d,) = lytk.check_lilypond(text) + assert (d.severity, d.code, d.line, d.column) == ("error", "syntax-error", 2, 9) + assert text[d.start : d.end] == ">>" # character offsets + assert str(d) == "2:9: error: `>>` without a matching `<<` [syntax-error]" + assert d == lytk.Diagnostic(*[getattr(d, k) for k in ("severity", "code", "message", "line", "column", "start", "end")]) + + def test_too_large_is_a_diagnostic_not_an_exception(self): + (d,) = lytk.check_lilypond("{ " * 3000 + "}" * 3000) + assert (d.severity, d.code) == ("error", "too-large") + + def test_readers_carry_diagnostics_outside_the_ir(self, tmp_path): + text = "\\include \"x.ily\"\n{ c'4 \\noSuchCommand d'4 }" + path = tmp_path / "in.ly" + path.write_text(text) + for music in ( + lytk.from_lilypond_string(text), + lytk.from_lilypond(str(path)), + lytk.from_lilypond_music_string(text), + lytk.from_lilypond_music(str(path)), + ): + assert [d.code for d in music.diagnostics] == ["ignored-include", "unknown-command"] + assert "diagnostic" not in music.to_json() + assert lytk.from_musicxml_string(lytk.to_musicxml(music.to_score())).diagnostics == [] + + def test_strict_raises_on_errors_only(self): + text = "{ c'4 \\noSuchCommand d'4 }" # a warning + assert lytk.from_lilypond_string(text, strict=True).num_parts == 1 + with pytest.raises(lytk.LilyPondSyntaxError, match=r"1:1: error: missing `}`") as info: + lytk.from_lilypond_string("{ c'4 d'4", strict=True) + assert [d.severity for d in info.value.diagnostics] == ["error"] + with pytest.raises(lytk.LilyPondSyntaxError, match=r"\(and 1 more error\)"): + lytk.from_lilypond_music_string("{ c'3 d'3 }", strict=True) + # Without strict the same input reads, reporting what it could not. + assert lytk.from_lilypond_string("{ c'3 d'3 }").diagnostics[0].code == "invalid-duration" + + def test_movements_and_the_first_movement(self, tmp_path): + path = tmp_path / "two.ly" + path.write_text("\\score { { c'1 } }\n\\score { { d'1 } }\n") + movements = lytk.from_lilypond_movements(str(path), strict=True) + assert [len(m.diagnostics) for m in movements] == [0, 0] + (dropped,) = lytk.from_lilypond(str(path)).diagnostics + assert (dropped.code, dropped.line) == ("dropped-music", 2) + + +class TestStringsAndHeaders: + """Strings decoded as LilyPond reads them, headers as dicts (Epic J4); + pitch-language files (J5).""" + + def test_header_is_a_dict_of_every_field(self): + text = '\\header { title = "A \\"B\\"" composer = \\markup { \\bold "J. S." Bach } opus = "5" }\n{ c\'1 }' + score = lytk.from_lilypond_string(text) + assert score.header == {"title": 'A "B"', "composer": "J. S. Bach", "opus": "5"} + doc = lytk.from_lilypond_music_string('\\header { poet = "P" }\n{ c\'1 }') + assert (doc.lyricist, doc.header) == ("P", {"lyricist": "P"}) + + def test_each_movement_has_its_header(self, tmp_path): + path = tmp_path / "two.ly" + path.write_text('\\header { composer = "C" }\n\\score { \\header { piece = "I" } { c\'1 } }\n\\score { { d\'1 } }\n') + first, second = lytk.from_lilypond_movements(str(path)) + assert first.header == {"composer": "C", "piece": "I"} + assert second.header == {"composer": "C"} + + def test_header_fields_locate_each_assignment(self): + text = '% é\n\\header { title = "é\\"x" tagline = ##f }\n\\score { \\header { piece = "II" } { c\'1 } }' + fields = lytk.header_fields(text) + assert [(f.key, f.value, f.score) for f in fields] == [("title", 'é"x', None), ("piece", "II", 0)] + assert text[fields[0].start : fields[0].end] == 'title = "é\\"x"' # character offsets + cut = text[: fields[0].start] + text[fields[0].end :] + assert [f.key for f in lytk.header_fields(cut)] == ["piece"] + + def test_language_files_set_the_pitch_names(self): + score = lytk.from_lilypond_string('\\include "english.ly"\n{ cs\'4 }') + assert [n[2] for n in score.notes()] == [61] + assert score.diagnostics == [] + + +class TestPanicFirewall: + """A Rust panic reaches Python as lytk.InternalError, never as pyo3's + PanicException (a BaseException), and prints nothing.""" + + def test_hierarchy(self): + assert issubclass(lytk.InternalError, lytk.LytkError) + assert issubclass(lytk.LytkError, Exception) + + def test_panic_becomes_internal_error_silently(self, capfd): + with pytest.raises(lytk.InternalError, match=r"lytk panicked: boom \(at src/python\.rs:\d+\)"): + lytk._core._panic_for_tests("boom") + assert capfd.readouterr().err == "" + + def test_except_exception_catches_it(self): + try: + lytk._core._panic_for_tests("caught") + except Exception as e: # noqa: BLE001 - the point of the test + assert isinstance(e, lytk.InternalError) + else: + pytest.fail("no exception") + + def test_library_still_works_after_a_panic(self): + for _ in range(3): + with pytest.raises(lytk.InternalError): + lytk._core._panic_for_tests("again") + assert lytk.from_lilypond_string("{ c'4 d' }").num_parts == 1 + + + +def _set_first(node, key, value): + """Set the first occurrence of `key` in a nested JSON value; True if found.""" + if isinstance(node, dict): + if key in node: + node[key] = value + return True + return any(_set_first(v, key, value) for v in node.values()) + if isinstance(node, list): + return any(_set_first(v, key, value) for v in node) + return False + + +class TestHandSuppliedIr: + """from_dict / from_json refuse the values the IR would divide by.""" + + LY = r"{ \time 3/4 \tuplet 3/2 { c8 d e } f4 }" + + @pytest.mark.parametrize( + ("key", "value"), + [("beat_type", 0), ("tuplet_actual", 0), ("tuplet_normal", 0), ("base", [-1, 8])], + ) + def test_score_refuses_values_no_reader_makes(self, key, value): + d = lytk.from_lilypond_string(self.LY).to_dict() + assert _set_first(d, key, value) + with pytest.raises(ValueError, match="invalid IR"): + lytk.Score.from_dict(d) + with pytest.raises(ValueError, match="invalid IR"): + lytk.Score.from_json(json.dumps(d)) + + def test_music_document_refuses_a_zero_tuplet_term(self): + d = json.loads(lytk.from_lilypond_music_string(self.LY).to_json()) + with pytest.raises(ValueError, match="invalid IR"): + e = json.loads(json.dumps(d)) + assert _set_first(e, "tuplet_actual", 0) + lytk.MusicDocument.from_json(json.dumps(e)) + # A Tuplet node of the Music tree, as the ABC and Humdrum readers make them. + d["music"] = {"Tuplet": {"normal": 2, "actual": 0, "content": d["music"]}} + with pytest.raises(ValueError, match="invalid IR"): + lytk.MusicDocument.from_json(json.dumps(d)) + + def test_valid_ir_round_trips(self): + score = lytk.from_lilypond_string(self.LY) + assert lytk.Score.from_dict(score.to_dict()) == score + assert lytk.Score.from_json(score.to_json()) == score + + + +class TestArgumentBounds: + """Extreme arguments are a ValueError, not an i32 overflow in Rust.""" + + LY = r"{ c'4 d' e' f' }" + + def test_transpose(self): + score = lytk.from_lilypond_string(self.LY) + assert lytk.transpose(score, 127).num_parts == 1 + with pytest.raises(ValueError, match="semitones"): + lytk.transpose(score, 2**31 - 1) + + def test_invert_axis(self): + score = lytk.from_lilypond_string(self.LY) + with pytest.raises(ValueError, match="axis"): + lytk.invert(score, octave=2**31 - 1) + with pytest.raises(ValueError, match="axis"): + lytk.invert(score, alter=2**31 - 1) + + def test_interval_number(self): + score = lytk.from_lilypond_string(self.LY) + with pytest.raises(ValueError, match="between 1 and 99"): + lytk.transpose_interval(score, "P2147483647") + + def test_extreme_octave_in_hand_supplied_ir(self): + d = lytk.from_lilypond_string(self.LY).to_dict() + assert _set_first(d, "octave", 2**31 - 1) + with pytest.raises(ValueError, match="octave"): + lytk.Score.from_dict(d) + + # -- Review R8/R9: Layer-1 transforms, transpose_to_key, compressed MXL ------- diff --git a/tests/test_cli.py b/tests/test_cli.py index 6784e61..ba7a695 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -65,7 +65,7 @@ def test_help_lists_every_command(self): out = ok("--help").stdout for command in [ "convert", "transpose", "invert", "retrograde", "change-language", "abs2rel", - "rel2abs", "info", "positions", "bundle", "diff", "batch", "flatten", + "rel2abs", "info", "positions", "bundle", "diff", "batch", "flatten", "check", ]: assert command in out @@ -513,3 +513,30 @@ def test_include_path(self, tmp_path: Path): main = tmp_path / "main.ly" main.write_text('\\include "inc.ily"\n') assert "a'4" in ok("flatten", str(main), "-I", str(lib)).stdout + + +class TestCheck: + def test_clean_file(self, simple_ly: Path): + assert ok("check", str(simple_ly)).stdout == "" + + def test_errors_exit_one(self, tmp_path: Path): + bad = tmp_path / "bad.ly" + bad.write_text("{ c'4 >> }\n") + result = run("check", str(bad)) + assert result.exit_code == 1 + assert result.stdout == f"{bad}:1:7: error: `>>` without a matching `<<` [syntax-error]\n" + + def test_semantic_warnings_exit_zero(self, tmp_path: Path): + path = tmp_path / "w.ly" + path.write_text("{ c'4 \\noSuchCommand }\n") + assert ok("check", str(path)).stdout == "" + assert "[unknown-command]" in ok("check", "--semantic", str(path)).stdout + + def test_json_and_stdin(self): + result = run("check", "--json", "-", input="{ c'3 }\n") + assert result.exit_code == 0 # a semantic error needs --semantic + assert json.loads(result.stdout) == [] + result = run("check", "--json", "--semantic", "-", input="{ c'3 }\n") + assert result.exit_code == 1 + (row,) = json.loads(result.stdout) + assert row["file"] == "-" and row["code"] == "invalid-duration" and row["line"] == 1 diff --git a/uv.lock b/uv.lock index 23241af..5823900 100644 --- a/uv.lock +++ b/uv.lock @@ -594,7 +594,6 @@ wheels = [ [[package]] name = "lytk" -version = "0.2.0" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -642,7 +641,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'eval'", specifier = ">=4.30" }, { name = "typer", specifier = ">=0.26.8" }, ] -provides-extras = ["torch", "tensorflow", "eval", "all"] +provides-extras = ["all", "eval", "tensorflow", "torch"] [package.metadata.requires-dev] dev = [