diff --git a/Cargo.lock b/Cargo.lock index bac71e9..b444256 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,7 +457,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lytk" -version = "0.3.0" +version = "0.4.0" dependencies = [ "cc", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 416cb76..63eefc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lytk" -version = "0.3.0" +version = "0.4.0" edition = "2021" # Floor set by our oldest dependency (clap 4.6). Matters because the PyPI sdist # is compiled on the user's own toolchain. diff --git a/README.md b/README.md index 6fd51fc..1472a1d 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,9 @@ from the command line. release to the next. - **Built for machine learning.** Encoders for note arrays, piano rolls and Performance-RNN event sequences, objective metrics from the muspy family, - folder datasets with deterministic splits and caching, and padded data - loaders for PyTorch and TensorFlow. + datasets over folders or JSON Lines records with deterministic (or the + records' own) splits and caching, and padded data loaders for PyTorch and + TensorFlow. - **Fast.** A Rust core with prebuilt wheels, and a batch converter that uses every core. It transposes LilyPond about 50× faster than python-ly. @@ -108,6 +109,19 @@ Scores differ in length, so each batch is padded. The true lengths come back alongside it because `0` is a valid event, pitch and velocity, so padding alone can't tell you where a score ends. `to_tensorflow_dataloader` works the same way. +A corpus kept as JSON Lines records (id, text, metadata) keeps its ids and its +own splits: + +```python +from lytk.datasets import RecordsDataset + +data = RecordsDataset.from_jsonl("scores.jsonl", split_field="split", on_error="skip") +splits = data.split() # {"train": …, "valid": …, "test": …}, as recorded +loader = splits["train"].to_pytorch_dataloader("note_array", batch_size=32, return_ids=True) +for arrays, lengths, ids in loader: + ... +``` + The package also installs a `lytk` command: ```bash @@ -154,6 +168,7 @@ lyrics); slurs and ornaments are not performed. ## Documentation +- [Python API reference](docs/python-api.md) - [Command line](docs/cli.md) - [What each format reads and writes](docs/import-export.md) - [Design and internals](docs/design.md) diff --git a/SECURITY.md b/SECURITY.md index 0fa80b3..d54665f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,10 +19,13 @@ reachable from a malicious input file are in scope, in particular: - panics or unbounded memory/CPU use while parsing (a reachable denial-of-service for any service that converts user uploads) -- path traversal or unintended file reads. The LilyPond reader does not - follow `\include` (it reports it); `lytk flatten` / `lytk.flatten` does, by - design, so callers that flatten untrusted `.ly` files should run it in a - sandbox and pass only trusted `-I` search paths +- path traversal or unintended file reads. The LilyPond readers do not + follow `\include` (they report it) unless given `include_paths`; + `lytk flatten`, `lytk.flatten` and `lytk.flatten_string` do, by design. An + include may name any path, relative (`../../x`) or absolute, and the + file's text becomes part of the output, so callers that follow the + includes of untrusted `.ly` text should run in a sandbox and pass only + trusted search paths - decompression bombs in `.mxl` archives (these are read with a size cap) Release builds keep `overflow-checks` on, so an integer overflow panics rather diff --git a/docs/changelog.md b/docs/changelog.md index 56542a7..51297d4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,106 @@ engineering notes are in the [development log](devlog.md). ## [Unreleased] +## [0.4.0] - 2026-09-28 + +LilyPond source you can inspect and edit, and datasets for curated +corpora. `\version` is read, compared and edited; LilyPond text is +tokenized (embedded Scheme as Scheme) and counted; includes are found on +the syntax tree, `flatten_string` flattens text, and every reader follows +includes when given `include_paths=`, with diagnostics pointing into the +caller's text. `lytk.info` gives a score's statistics. Datasets read JSON +Lines records as well as folders, keep ids and a corpus's own splits, skip +what they cannot read when asked, and cache by content. The Python API +reference is generated from the stubs. + +### Added + +- LilyPond `\version`: `lytk.lilypond_version(text)` reads the version the + first `\version` statement states, from the parse tree (a commented-out + one does not count), as a `lytk.LilyPondVersion`, which compares + numerically (`2.24` equals `2.24.0`) and accepts what LilyPond 2.24 + accepts. `lytk.set_lilypond_version(text, version)` and + `lytk.strip_lilypond_version(text)` edit the statements (a version given + as a string is written as given, once checked). + `Score.lilypond_version` and `MusicDocument.lilypond_version` give the + version a LilyPond source stated. `to_lilypond(…, version=)` and + `to_lilypond_music(…, version=)` write another `\version` than 2.24.0. + An invalid `\version` is an `invalid-version` error. Rust: + `ly_to_ir::{LilyPondVersion, lilypond_version, set_lilypond_version, + strip_lilypond_version}` and `ScoreMetadata::lilypond_version`. +- `lytk.tokenize(text)`: the tokens of LilyPond text from the parse tree + (`lytk.Token`: kind, text, character span, line, column, and whether it + is Scheme). The kinds are comment, string, command, symbol, number, + fraction, punctuation, and error for text the grammar cannot tokenize, so + every character but whitespace is in exactly one token, broken input + included. Embedded Scheme is tokenized as Scheme: `scheme` is the `#` or + `$` that starts it, then its symbols, numbers, strings, comments, + brackets and quotes, with `boolean`, `character` and `keyword` for + Scheme's own; LilyPond inside it (`#{ … #}`) is tokenized as LilyPond. + `lytk.strip_comments(text)` removes LilyPond's and Scheme's comments. + Rust: `ly_to_ir::{tokenize, strip_comments, Token, TokenKind}`. +- `lytk.info(score)`: a score's metadata and counts as a dict, the library + home of `lytk info --json`, which now calls it. Besides the parts and notes + it counts voices, bars, the length in quarter notes, lyric syllables, chord + symbols and grace notes, computed in Rust. `lytk.source_stats(text)`: + bytes, lines, tokens, comments, Scheme expressions and error tokens of + LilyPond text. Rust: `ir::stats::score_info`, `ly_to_ir::source_stats`. +- `lytk.from_lilypond_music_movements(path)`: every movement of a LilyPond + file as a `MusicDocument`, as `from_lilypond_movements` gives them as + scores; `lytk.from_lilypond_movements_string(text)`, every movement of + LilyPond text. +- `lytk.flatten_string(text, *, base_dir=None, include_paths=None, + add_markers=True)`: `flatten` for text. `lytk.from_abc_tunes_string(text)`: + every tune of ABC text. +- `lytk.datasets.RecordsDataset`: a dataset over records holding music as + text, `from_jsonl(path, …)` or `from_records(iterable, …)`, with + `text_field`, `id_field`, `format` (LilyPond, MusicXML, ABC or `**kern`) + and `split_field`. Items keep their ids and records (`record(i)`); with + `split_field`, `split()` returns the records' own splits as + `{value: Subset}` instead of re-shuffling them. +- Datasets: `on_error="raise" | "skip" | "warn"` (iteration leaves out an + item that cannot be read and records it in `dataset.errors`; indexing + still raises); `movements="all"` (each movement of a file or record is an + item); `language`, `include_paths`, `strict` and MIDI `quantize` passed to + the readers; `ids` on every dataset (a folder's are paths relative to it); + `return_ids=True` on the torch and TensorFlow datasets and data loaders; + `split(…, groups=…)` keeps the items of a group in one subset. +- `docs/python-api.md`, the Python API reference, generated from the stubs + (`scripts/python_api.py`); a test fails when it is stale. +- `include_paths=` on every LilyPond reader and on `check_lilypond`: given + (possibly empty), `\include`s are followed, relative to the file's + directory, then the paths. A diagnostic in an included file is reported at + its `\include`, naming the file; an include not found is an + `ignored-include` warning. Without it, includes are still not followed. + Rust: `LyToIrAdapter::with_include_paths`, `read_text` and `check_str`. + +### Changed + +- `flatten` finds includes on the parse tree: anywhere in a line, never in a + comment or a string. It used to miss an include not at the start of a + line, and after a one-line `%{ … %}` it took the rest of the file for a + comment and followed no include in it. +- `flatten` keeps an include of LilyPond's own files (`english.ly`, + `gregorian.ly`, …) as it is when no such file is found; it was a "not + found" error. Rust: `flatten_str` takes `Option<&Path>` for its base + directory, and `FlattenOpts::keep_missing` keeps any missing include. +- The dataset cache is keyed by the item's content, how it is read, lytk's + version and the converter's arguments with their defaults filled in, and + written atomically: an edited file is no longer served stale, and + `to_note_arrays()` and `iter_representation("note_array")` share entries. + Entries of earlier versions are not read again. A `Subset` uses its + parent's cache (it bypassed it). + +### Fixed + +- `flatten` kept only the last `\language` line, so music written before it + was read in the wrong language; every `\language` now stays. +- `flatten` refused a file with more than one `\header` block, a top-level + one with a score's own included (`FlattenError::MultipleHeaders`, now + gone). LilyPond merges top-level headers, and so does lytk's reader. +- The `ignored-include` warning for one of LilyPond's own files (other than a + language file) no longer advises flattening, which would not follow it. + ## [0.3.0] - 2026-09-27 LilyPond input you can trust. Readers raise `lytk.ParseError` instead of @@ -477,7 +577,8 @@ First public release. newer. - MIT licence. -[Unreleased]: https://github.com/CSCPadova/lytk/compare/v0.3.0...HEAD +[Unreleased]: https://github.com/CSCPadova/lytk/compare/v0.4.0...HEAD +[0.4.0]: https://github.com/CSCPadova/lytk/compare/v0.3.0...v0.4.0 [0.3.0]: https://github.com/CSCPadova/lytk/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/CSCPadova/lytk/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/CSCPadova/lytk/releases/tag/v0.1.0 diff --git a/docs/cli.md b/docs/cli.md index b6f0042..5b373a2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -152,9 +152,12 @@ Parts: 1 - MusicXML Part (28 measures) ``` -`--json` prints title, subtitle, composer, arranger, lyricist, language, -`part_count`, `note_count`, and for each part its id, name, abbreviation, -number of measures, staves, MIDI program and MIDI instrument. +`--json` prints `lytk.info(score)`: title, subtitle, composer, arranger, +lyricist, language, `lilypond_version`, `part_count`, `note_count`, +`voice_count`, `bar_count`, `duration_quarters`, `lyric_count`, +`chord_symbol_count`, `grace_note_count`, and for each part its id, name, +abbreviation, number of measures, staves, MIDI program, MIDI instrument, +voices and notes. ## positions @@ -226,13 +229,17 @@ lytk flatten INPUT.ly [-o OUTPUT] [-I DIR ...] [--no-markers] ``` Inlines every `\include`, recursively, into one self-contained file (stdout -without `-o`). Includes are looked up next to the file that includes them, then -with `.ly` and `.ily` appended, then in each `-I`/`--include-path` directory. -A missing file and an include cycle are errors; the cycle is shown -(`a.ly -> b.ly -> a.ly`). - -Repeated `\version` and `\language` lines are merged, keeping the last one and -warning on stderr. More than one `\header` block is an error. Each inlined file +without `-o`). Includes are found on the parse tree, anywhere in a line and +never in a comment or string. They are looked up next to the file that +includes them, then with `.ly` and `.ily` appended, then in each +`-I`/`--include-path` directory. A missing file and an include cycle are +errors; the cycle is shown (`a.ly -> b.ly -> a.ly`). An include of one of +LilyPond's own files (`english.ly`, `gregorian.ly`) stays as it is when no +such file is found: LilyPond finds it in its installation. + +Repeated `\version` lines are merged, keeping the last one and warning on +stderr. `\language` lines and `\header` blocks all stay: each `\language` +applies from where it stands, and LilyPond merges headers. Each inlined file is wrapped in `% === BEGIN INCLUDE: … ===` / `% === END INCLUDE: … ===` comments; `--no-markers` leaves them out. diff --git a/docs/devlog.md b/docs/devlog.md index 77b4bd5..686e62b 100644 --- a/docs/devlog.md +++ b/docs/devlog.md @@ -3,6 +3,188 @@ Dated engineering notes, newest first. The release history is in [changelog.md](changelog.md). +## 2026-09-28 — Release 0.4.0 + +Epics K and L ship as **0.4.0**, with the Scheme tokens and +`from_lilypond_movements_string` added after K. The version is bumped in +`Cargo.toml`, its only source (and in `Cargo.lock`); `[Unreleased]` became +`[0.4.0] - 2026-09-28`. As for 0.3.0: a pull request to `master`, the tag on +the merge commit, the release workflow publishing to PyPI, and a GitHub +release with the changelog section. The next minor release is P5, which +breaks the Python API (0.5.0). Just before tagging, a version given as a +string to `set_lilypond_version` or `version=` became written as given +(checked, not canonicalized), so lilycorpus's synthesized +`\version "2.24"` keeps its text, and its records their ids. + +## 2026-09-28 — Epic L (datasets), shipped with K in 0.4.0 + +The owner chose to release Epic L in 0.4.0 together with K; it was planned +for 0.5.0. + +The datasets module was rebuilt around one private base, `_SourceDataset`, +shared by `FolderDataset` (a source is a file) and the new `RecordsDataset` +(a source is a record's text). A source's reading goes through one +`_read`, which passes the LilyPond options (`language`, `include_paths`, +`strict`) to LilyPond only and `quantize` to MIDI only, and returns the first +movement or all of them. `movements="all"` reads every source once to count +its movements, keeps only the counts, and reads a source again for its items +(the last source read is kept, so a sequential pass reads each twice). + +`on_error` applies where the dataset iterates: `for doc in ds`, +`iter_representation`, `to_representation`, `metrics` and the TensorFlow +generator (whose probe item is now the first readable one). Indexing +(`ds[i]`, the torch dataset) asks for one item and raises: a torch dataset +cannot drop an index. Skipped items go to `errors`, keyed by id; a +`Subset` records into its parent's. + +The cache key used to be the path and the arguments as given: an edited +file was served stale, and `to_note_arrays()` (which passes `resolution=480`) +and `iter_representation("note_array")` (which passes nothing) wrote two +files. It is now a SHA-256 of the source's bytes, the reading options, the +movement, lytk's version and the converter's arguments with the defaults +filled in; files are written to a temporary name and renamed. `Subset` +now delegates its conversion to its parent, so it uses the cache (it +converted without it). Included files' content is not in the key. + +`RecordsDataset.split()` returns `{value: Subset}` for the split field (or +any `field=`), in order of first appearance, so a curated split is never +re-shuffled by accident; passing ratios is a deliberate re-split. Ratio +splits take `groups`: the groups are shuffled with the seed and each goes to +the subset furthest below its share, which keeps a dedup cluster (or a +folder) in one subset. Without `groups` the split is the old one, item for +item. Ids: paths relative to the root, records' ids, `#k` for movements; +`return_ids` adds them to torch and TensorFlow items and batches +(`pad_collate` passes them through). The torch and TensorFlow paths are +tested in CI only (neither is installed locally). + +`docs/python-api.md` is written by `scripts/python_api.py` from the stubs, +in the sections of `lytk.__all__`, plus the datasets module; private bases +are folded into the public classes. Two tests keep it honest: the file must +equal a fresh rendering, and every public name of the compiled extension +must have a stub. + +Tests: 1,158 Rust, 262 Python (14 skipped without torch and TensorFlow). + +## 2026-09-28 — Scheme tokens + +K3 left embedded Scheme as one token, noting that the compiled Scheme +grammar could split it. It turned out not to be needed: the main grammar +already parses embedded Scheme into `scheme_*` nodes (lists, symbols, +numbers, strings, booleans, characters, keywords, quotes, vectors, +comments) and LilyPond embedded in Scheme (`#{ … #}`) into LilyPond nodes. +`tokenize` now descends into them, tracking with a stack of ancestors +whether a node is Scheme (inside `embedded_scheme`) or LilyPond again +(inside `scheme_embedded_lilypond_text`), and sets `Token.scheme`. The +Scheme kinds reuse the lexical ones where they mean the same (symbol, +number, string, comment, punctuation); `boolean`, `character` and `keyword` +are Scheme's own, and `scheme` is now the `#` or `$` that opens an +expression, so `source_stats` still counts expressions. A Scheme block +comment's text is in no child node, so a comment is one token, as a string +is. `strip_comments` now strips Scheme's comments as well. Nothing but +whitespace is lost, still: 0 of the corpus board's 2,706 files. The grammar +has two known gaps, which the tokens follow: a datum comment (`#;`) runs to +the end of its line, and `#| … |#` outside a Scheme expression is not a +comment. + +## 2026-09-28 — Epic K: K2 (includes) + +`flatten` found includes line by line: `\include` had to start a line, and a +line starting with `%{` switched it into a comment until a line starting +with `%}`, so a one-line `%{ … %}` hid every include after it. It now finds +them on the tree (`include_statements`), which also keeps them out of +strings. An include alone on its line replaces the line, as before; one +mid-line is spliced in on lines of its own. + +Two normalizations were wrong. Keeping only the last `\language` changes +the language of the music written before it; they all stay now. Refusing +several `\header` blocks rejected valid LilyPond: a second top-level header +copies the first and adds to it (`get_header` in `lily/lily-parser.cc`), +which lytk's reader already does; the error is gone. An include of one of +LilyPond's own files failed as "not found" unless LilyPond's `ly/` +directory was a search path; `scripts/ly_builtins.py` now also lists that +directory (62 files, `LILYPOND_FILES`), and such an include stays as it is. + +The readers take `include_paths=`. They flatten with missing files kept +(the walk warns about them, now saying they are not found), read the flat +text, and map each diagnostic back through the pieces `flatten_mapped` +records: text of the source keeps its place, and a diagnostic in an +included file moves to the `\include` that brought the file in, its message +naming it. Positions therefore always refer to the text the caller has. +Two diagnostics of one code in one included file then share a span, and +`into_first` keeps one of them. Without `include_paths` nothing changes: +includes are not followed, as SECURITY.md says, which now also warns that +an include may name any path. + +Tests: 1,157 Rust, 241 Python (`TestIncludes`). + +## 2026-09-28 — Epic K: K4 (statistics) and K5 (music movements) + +`lytk.info(score)` is the dict `lytk info --json` printed, now built in the +library and computed in Rust (`ir/stats.rs`). The CLI built it in Python +through the navigation objects, one Python object per note; a corpus that +wants statistics per record (lilycorpus's wishlist item 7) should not pay +that. The keys the CLI printed stay as they were, and `test_info_json` +checks the CLI prints exactly `lytk.info`. The new counts: voices (distinct +voice numbers per part, summed over parts), bars (the longest part's), +length in quarter notes (each bar as long as its longest voice, grace notes +taking no time), lyric syllables, chord symbols and grace notes. +`source_stats(text)` counts tokens by kind, with lines and error tokens +added to what the roadmap asked for. + +K5's first half, single-score readers reporting the movements they drop, +came with J3 and J7; `from_lilypond_music_movements` is the Music-tree twin +of `from_lilypond_movements`. + +Tests: 1,153 Rust, 239 Python (`TestStatistics`, `test_music_movements`). + +## 2026-09-28 — Epic K: K3 (tokens) + +`ly_to_ir/tokens.rs` walks the tree and makes a token of each leaf, and of +three composites kept whole: strings, embedded Scheme and quoted +identifiers. Anonymous leaves starting with a backslash (`\<`, `\!`, +`\(`) are commands, the other anonymous leaves punctuation. An ERROR leaf +is an `error` token, so the tokens of broken input still cover it; MISSING +nodes are empty and make none. The property that matters to callers such +as lilycorpus's dedup is that nothing but whitespace is lost: +`tests/common::token_gap` checks it on the fixtures and on broken input, +and the syntax board on the whole corpus (0 of 2,706 files lose text). + +`strip_comments` keeps what LilyPond reads: a block comment between two +tokens becomes a space (`c4%{x%}d4` is two notes in LilyPond), and a line +comment leaves its line break. Scheme's own comments stay, being part of a +Scheme token. Scheme is not sub-tokenized: the roadmap left it optional, +and nothing asks for it. + +Tests: 1,152 Rust, 236 Python (`TestTokens`). + +## 2026-09-28 — Epic K: K1 (`\version`) + +`ly_to_ir/version.rs` holds `LilyPondVersion` and one scan of the tree for +`\version` statements, which the reader (the version goes into every +movement's metadata), the syntax check and the three text functions +(`lilypond_version`, `set_lilypond_version`, `strip_lilypond_version`) all +use. A statement is the command and the string right after it; LilyPond's +lexer allows only whitespace between them, so a comment there makes the +statement invalid, as in LilyPond. + +What a version is follows LilyPond 2.24 (`parse-lily-version` in +`scm/lily-library.scm`, since 2.23.8): `x.y.z`, with a free fourth part that +is not compared, or `x.y` for an even `y` only, because within a +development series syntax changes with point releases. LilyPond's lexer +rejects a bad string and a missing one alike, so both are the new +`invalid-version` error. The "program too old" check is LilyPond's own and +is left out. `Score.lilypond_version` is informational: the writers keep +writing `2.24.0`, the syntax they write, unless `version=` says otherwise. +The IR field is skipped in JSON when unset, so existing JSON is unchanged. + +The corpus boards now leave out LilyPond's tests of its own errors +(`expect-error = ##t`, 13 files): LilyPond fails them on purpose. lytk +reports an error in 2 of them, the two version tests; before K1 it found +none. The boards ran on the 2.27.3 checkout this time (the 2.26 one was +gone): still the one false positive (`##[ #]`). + +Tests: 1,148 Rust (`tests/ly_source.rs`), 234 Python (`TestLilyPondVersion`). + ## 2026-09-27 — Release 0.3.0 Epic J (J0–J7) ships as **0.3.0**. The version was already 0.3.0 in diff --git a/docs/import-export.md b/docs/import-export.md index 3d7f9b3..05c7861 100644 --- a/docs/import-export.md +++ b/docs/import-export.md @@ -178,7 +178,7 @@ Layer 1 (`MusicDocument`, the Music tree the ML representations consume). | Multi-staff | ✅ | `\new Staff`, `\new PianoStaff` | | Variables | ✅ | Definition + resolution | | Movements | ✅ | Each `\score` block and each top-level music expression (`{ }`, `<< >>`, `\new Staff`, `\relative`, a music variable), in order, as LilyPond makes a score of each; `\book`/`\bookpart` scores and music too, their `\header` the book's | -| `\include` | 🟡 | LilyPond's language files (`english.ly`, …) set the pitch language; other files are not followed (`ignored-include`: flatten first) | +| `\include` | ✅ | LilyPond's language files (`english.ly`, …) set the pitch language. Other files are followed with `include_paths=` (the file's directory, then the paths; a diagnostic in an included file is reported at its `\include`); without it they are not (`ignored-include`) | | `\transpose` | ✅ | Transposing instrument context | | Relative mode | ✅ | `\relative` pitch context | | `\fixed` | ✅ | Absolute pitches an octave up per mark of the reference, also inside `\relative` | @@ -188,6 +188,8 @@ Layer 1 (`MusicDocument`, the Music tree the ML representations consume). | Slide | ✅ | Via glissando trill style | | Anacrusis | ✅ | `\partial` | | Paper block | ✅ | `\paper { }` → page layout | +| Tokens | ✅ | `lytk.tokenize` (the tree's leaves; strings and comments whole; embedded Scheme as Scheme tokens; nothing but whitespace lost) and `lytk.strip_comments` | +| `\version` | ✅ | `Score.lilypond_version` (every movement's); `lytk.lilypond_version`, `set_lilypond_version`, `strip_lilypond_version` read and edit it on the tree; an invalid one is `invalid-version` | | `\header` | ✅ | Every field (`Score.header`); strings decoded as LilyPond does; `\markup` values as plain text, `#"…"` as the string; a `\score`'s header is its movement's, the top-level one every movement's | | `\language` | ✅ | At the top level, in `\score` and in music; an unknown name keeps the language in force | | `\set Staff.instrumentName` | ✅ | Part name from `\set` property | @@ -218,9 +220,10 @@ A warning is input lytk reads around. | `invalid-ratio` | error | A zero term in a tuplet (`\tuplet 0/2`), a duration multiplier (`*1/0`) or a measure length | | `not-lilypond` | error | Words at the start of the file that are no LilyPond (plain text) | | `too-large` | error | Input past the reader's bounds (`check_lilypond` only; a reader raises `ParseError`) | +| `invalid-version` | error | `\version` without a quoted string after it, or with no version LilyPond 2.24 accepts (`"2.x"`; `"2.25"`, which leaves out the patch of a development release) | | `unknown-command` | warning | A command neither LilyPond 2.26 nor the file defines (a typo, or one from an include) | | `unrecognized-token` | warning | A word in music that is no note name in the current language, or no drum name in drum mode | -| `ignored-include` | warning | `\include`: the reader does not follow includes; flatten the file first | +| `ignored-include` | warning | `\include` of a file not read: includes are not followed without `include_paths`; with them, the file is not found; LilyPond's own files other than the language files are not read | | `unknown-language` | warning | `\language` with a name lytk does not know | | `dropped-music` | warning | The movements after the first, from a reader that takes one (`from_lilypond`, `convert_str`) | | `skipped-score` | warning | A `\score` with `\midi` and no `\layout` | @@ -237,6 +240,7 @@ Scheme parenthesis, 499 have one. | Feature | Status | Notes | |---|---|---| +| `\version` | ✅ | `2.24.0`, the syntax lytk writes, whatever the score was read from; `version=` on `to_lilypond`/`to_lilypond_music` writes another | | Notes with pitch | ✅ | All 12 languages, in the spelling LilyPond uses; relative/absolute | | Rests (r, R, s) | ✅ | | | Chords (`< >`) | ✅ | With arpeggio support | diff --git a/docs/python-api.md b/docs/python-api.md new file mode 100644 index 0000000..8e25888 --- /dev/null +++ b/docs/python-api.md @@ -0,0 +1,852 @@ +# Python API reference + +*Generated from `src/lytk/_core.pyi` and `src/lytk/datasets/` by +`python scripts/python_api.py`; a test fails when this file is out of date.* + +`import lytk` gives everything below; `lytk.__version__` is the version. + +- **Errors.** Readers (`from_*`, `Score.from_json`/`from_dict`, + `MusicDocument.from_json`, `flatten`) raise `lytk.ParseError`, a + `ValueError`, when the input cannot be read, and `OSError` when a file + cannot be opened. `strict=True` LilyPond readers raise + `lytk.LilyPondSyntaxError`, a `ParseError`, when the reading has an error. + Writers (`to_*`) raise `ValueError` when a score cannot be written. Any + function may raise `lytk.InternalError`, a bug in lytk. +- **Positions** (`Diagnostic`, `HeaderField`, `Token`) are character + offsets: `text[d.start:d.end]` is the span. +- **One LilyPond parse.** `from_lilypond_string` and + `from_lilypond_music_string` (and their file twins) run the same parse, + walk and assembly: the Music tree of `MusicDocument` is lifted from the + `Score`. Whatever one accepts, the other accepts too. + +## Errors and diagnostics + +### class `LytkError(Exception)` + +Base class of the errors lytk raises itself. + +### class `ParseError(LytkError, ValueError)` + +The input could not be read: malformed, not the format expected, or +past the reader's bounds. A ValueError too. + +### class `LilyPondSyntaxError(ParseError)` + +LilyPond read with ``strict=True`` has errors. + +### class `InternalError(LytkError)` + +A bug in lytk: the Rust code panicked. Every function and method that +reads, writes or transforms music raises this instead of letting a Rust +panic through; the message names the panic and where it happened. + +### class `Diagnostic` + +One finding of the LilyPond reader. Comparable, hashable, picklable. + +#### `Diagnostic(severity: str, code: str, message: str, line: int, column: int, start: int, end: int)` + +#### `Diagnostic.severity: str` + +``"error"``: LilyPond rejects the input; ``"warning"``: lytk does not +read it, or cannot represent it, and reads around it. + +#### `Diagnostic.code: str` + +A stable identifier: ``syntax-error``, ``missing-token``, +``invalid-duration``, ``invalid-ratio``, ``not-lilypond``, +``too-large``, ``invalid-version`` (errors); ``unknown-command``, ``unrecognized-token``, +``ignored-include``, ``unknown-language``, ``dropped-music``, +``skipped-score``, ``unsupported-value`` (warnings). + +#### `Diagnostic.message: str` + +#### `Diagnostic.line: int` + +1-based. + +#### `Diagnostic.column: int` + +1-based, in characters. + +#### `Diagnostic.start: int` + +Character offset: ``text[d.start:d.end]`` is the span. + +#### `Diagnostic.end: int` + +### `check_lilypond(text: str, *, semantic: bool=False, include_paths: list[str] | None=None) -> list[Diagnostic]` + +Diagnostics of LilyPond text, in source order: the syntax only (fast, +nothing is read), or with ``semantic=True`` what a reading reports too. +Never raises for bad input: input too large to read is a ``too-large`` +error. *include_paths* follows includes as the readers do; an included +file that cannot be read raises ``OSError``. + +### class `HeaderField` + +A field of a LilyPond ``\header`` block (see :func:`header_fields`). + +#### `HeaderField.key: str` + +#### `HeaderField.value: str` + +The value as text: strings decoded, ``\markup`` as its words, +``#"…"`` as its string. + +#### `HeaderField.start: int` + +Character offset: ``text[f.start:f.end]`` is the whole ``key = value``. + +#### `HeaderField.end: int` + +#### `HeaderField.score: int | None` + +Index of the ``\score`` block the field is in (file order), or +``None`` at the top level. + +### `header_fields(text: str) -> list[HeaderField]` + +Every ``\header`` field of LilyPond text, from the parse tree alone +(nothing is read); fields with values other than text (``##f``) are left +out. + + +## LilyPond \version + +### class `LilyPondVersion` + +A LilyPond version, compared numerically (``2.24`` equals ``2.24.0``). +Accepted as LilyPond 2.24 accepts it: ``major.minor.patch`` with an +optional fourth part (kept, not compared), or ``major.minor`` with an even +minor. Other strings raise :class:`ParseError`. Hashable, picklable. + +#### `LilyPondVersion(text: str)` + +#### `LilyPondVersion.major: int` + +#### `LilyPondVersion.minor: int` + +#### `LilyPondVersion.patch: int` + +#### `LilyPondVersion.extra: str | None` + +The fourth part, if any. + +### `lilypond_version(text: str) -> LilyPondVersion | None` + +The version the first ``\version`` statement states, from the parse +tree (a commented-out one does not count); *None* when there is none or +it is invalid (``check_lilypond`` reports ``invalid-version``). + +### `set_lilypond_version(text: str, version: LilyPondVersion | str) -> str` + +*text* with every ``\version`` statement stating *version* (a string +is written as given, once checked), or with one added at the top. + +### `strip_lilypond_version(text: str) -> str` + +*text* without its ``\version`` statements. + + +## LilyPond tokens + +### class `Token` + +A token of LilyPond text (see :func:`tokenize`). Hashable. + +#### `Token.kind: str` + +``comment``, ``string``, ``command``, ``symbol``, ``number``, +``fraction``, ``punctuation``, ``scheme`` (the ``#`` or ``$`` that +starts embedded Scheme), ``boolean``, ``character``, ``keyword`` +(Scheme's) or ``error``. + +#### `Token.scheme: bool` + +Whether the token is Scheme rather than LilyPond. + +#### `Token.text: str` + +#### `Token.start: int` + +Character offset: ``text[t.start:t.end]`` is the token. + +#### `Token.end: int` + +#### `Token.line: int` + +1-based. + +#### `Token.column: int` + +1-based, in characters. + +### `tokenize(text: str) -> list[Token]` + +The tokens of LilyPond text, from the parse tree; strings and comments +whole, embedded Scheme as Scheme tokens. Every character but whitespace +is in exactly one token, ``error`` tokens holding what the grammar cannot +tokenize. + +### `strip_comments(text: str) -> str` + +*text* without its comments, LilyPond's and embedded Scheme's. + + +## Statistics + +### `info(score: Score) -> dict[str, Any]` + +A score's metadata and counts, as ``lytk info --json`` prints them: +``title`` … ``language``, ``lilypond_version``; ``part_count``, +``note_count``, ``voice_count``, ``bar_count``, ``duration_quarters``, +``lyric_count``, ``chord_symbol_count``, ``grace_note_count``; and +``parts``, a dict per part. + +### `source_stats(text: str) -> dict[str, int]` + +Counts of LilyPond source text from its tokens: ``bytes``, ``lines``, +``tokens``, ``comments``, ``scheme``, ``error_tokens``. + + +## Scores + +### class `MusicDocument` + +Opaque handle to a parsed music document (Layer 1 Music tree). + +#### `MusicDocument.title: str | None` + +#### `MusicDocument.composer: str | None` + +#### `MusicDocument.subtitle: str | None` + +#### `MusicDocument.arranger: str | None` + +#### `MusicDocument.language: str | None` + +#### `MusicDocument.diagnostics: list[Diagnostic]` + +What reading LilyPond reported (empty for other sources). + +#### `MusicDocument.lyricist: str | None` + +#### `MusicDocument.lilypond_version: LilyPondVersion | None` + +As :attr:`Score.lilypond_version`. + +#### `MusicDocument.header: dict[str, str]` + +Every header field, as :attr:`Score.header`. + +#### `MusicDocument.notes(resolution: int=480) -> list[tuple[int, int, int, int]]` + +Notes as ``(onset, duration, pitch, velocity)`` tuples in time steps. + +#### `MusicDocument.to_json() -> str` + +#### static `MusicDocument.from_json(json: str) -> MusicDocument` + +#### `MusicDocument.to_score() -> Score` + +### class `Score` + +Opaque handle to a parsed music score. + +#### `Score.title: str | None` + +#### `Score.composer: str | None` + +#### `Score.subtitle: str | None` + +#### `Score.arranger: str | None` + +#### `Score.lyricist: str | None` + +#### `Score.lilypond_version: LilyPondVersion | None` + +The version the source's ``\version`` states, for a score read +from LilyPond that states a valid one. + +#### `Score.language: str | None` + +#### `Score.num_parts: int` + +#### `Score.parts: list[str]` + +#### `Score.diagnostics: list[Diagnostic]` + +What reading LilyPond reported (empty for other sources); not part +of ``to_dict``/``to_json``. + +#### `Score.header: dict[str, str]` + +Every header field: ``title``, ``subtitle``, ``composer``, +``arranger`` and ``lyricist`` (LilyPond's ``poet``) when set, then the +others (``copyright``, ``opus``, ``texidoc``, …) by key. + +#### `Score.to_json() -> str` + +#### static `Score.from_json(json: str) -> Score` + +#### `Score.to_dict() -> dict[str, Any]` + +#### static `Score.from_dict(dict: dict[str, Any]) -> Score` + +#### `Score.to_music_document() -> MusicDocument` + +#### `Score.notes(resolution: int=480) -> list[tuple[int, int, int, int]]` + +Notes as ``(onset, duration, pitch, velocity)`` tuples in time steps. + +#### `Score.iter_parts() -> list[Part]` + +Structured navigation: parts as typed :class:`Part` objects. + + +## Structured note navigation (read-only Layer-2 tree) + +### class `Part` + +An instrument part — a sequence of measures (read-only). + +#### `Part.name: str` + +#### `Part.abbreviation: str` + +#### `Part.part_id: str` + +#### `Part.midi_program: int` + +#### `Part.midi_instrument: str` + +#### `Part.midi_channel: int` + +#### `Part.staves: int` + +#### `Part.measures: list[Measure]` + +#### `Part.notes: list[Note]` + +Every sounding note in the part. + +### class `Measure` + +A measure / bar (read-only). + +#### `Measure.number: int` + +#### `Measure.number_label: str | None` + +#### `Measure.implicit: bool` + +#### `Measure.senza_misura: bool` + +#### `Measure.time_signature: tuple[str, int, str | None] | None` + +``(beats, beat_type, symbol)`` if set at this measure, else None. + +#### `Measure.key_signature: tuple[int, str] | None` + +``(fifths, mode)`` if set at this measure, else None. + +#### `Measure.voices: list[Voice]` + +#### `Measure.notes: list[Note]` + +### class `Voice` + +A voice within a measure (read-only). + +#### `Voice.number: int` + +#### `Voice.elements: list[Note | Rest | Chord]` + +#### `Voice.notes: list[Note]` + +Every sounding note (chord members flattened). + +### class `Note` + +A single pitched note (read-only). + +#### `Note.pitch: Pitch` + +#### `Note.midi: int` + +#### `Note.duration: float` + +Duration in quarter-note lengths. + +#### `Note.duration_fraction: tuple[int, int]` + +Exact duration as ``(numerator, denominator)`` of a whole note. + +#### `Note.voice: int` + +#### `Note.staff: int` + +#### `Note.is_grace: bool` + +#### `Note.velocity: int | None` + +MIDI velocity the note was played with, when known. + +#### `Note.ties: list[str]` + +Tie events: ``"start"`` / ``"stop"`` / ``"continue"``. + +#### `Note.articulations: list[str]` + +#### `Note.lyrics: list[str]` + +### class `Rest` + +A rest or spacer (read-only). + +#### `Rest.duration: float` + +#### `Rest.duration_fraction: tuple[int, int]` + +#### `Rest.voice: int` + +#### `Rest.staff: int` + +#### `Rest.is_measure_rest: bool` + +#### `Rest.is_spacer: bool` + +### class `Chord` + +A chord — notes sounding together (read-only). + +#### `Chord.duration: float` + +#### `Chord.duration_fraction: tuple[int, int]` + +#### `Chord.voice: int` + +#### `Chord.staff: int` + +#### `Chord.notes: list[Note]` + +### class `Pitch` + +A note's pitch (read-only). + +#### `Pitch.step: str` + +Diatonic step name, e.g. ``"C"``. + +#### `Pitch.alter: int` + +Chromatic alteration in semitones (1 = sharp, -1 = flat). + +#### `Pitch.octave: int` + +#### `Pitch.midi: int` + +MIDI note number (middle C = 60). + +#### `Pitch.name: str` + +Scientific pitch name, e.g. ``"C#4"``. + + +## Adapters + +### `from_musicxml(path: str) -> Score` + +### `from_musicxml_string(xml: str) -> Score` + +### `from_musicxml_bytes(data: bytes) -> Score` + +Parse MusicXML or compressed MXL from in-memory bytes. + +### `from_lilypond(path: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> Score` + +The first movement; ``strict=True`` raises LilyPondSyntaxError on an +error. The diagnostics are in ``Score.diagnostics`` either way. With +*include_paths*, ``\include`` statements are followed (the file's directory, +then the paths); a diagnostic in an included file is reported at its +``\include``, and one not found is an ``ignored-include`` warning. + +### `from_lilypond_string(text: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> Score` + +### `from_lilypond_movements(path: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> list[Score]` + +Every movement of a LilyPond file: one score per ``\score`` block and +per top-level music expression, each with the file's diagnostics. + +### `from_lilypond_movements_string(text: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> list[Score]` + +Every movement of LilyPond text, each with the text's diagnostics. + +### `from_lilypond_music(path: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> MusicDocument` + +### `from_lilypond_music_movements(path: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> list[MusicDocument]` + +Every movement of a LilyPond file as a Music tree, each with the +file's diagnostics. + +### `from_lilypond_music_string(text: str, *, language: str | None=None, strict: bool=False, include_paths: list[str] | None=None) -> MusicDocument` + +### `from_abc(path: str) -> Score` + +### `from_humdrum(path: str) -> Score` + +### `from_abc_string(text: str) -> Score` + +### `from_abc_tunes(path: str) -> list[Score]` + +Every tune of an ABC file, one score each (``from_abc`` reads the first). + +### `from_abc_tunes_string(text: str) -> list[Score]` + +Every tune of an ABC string, one score each. + +### `from_humdrum_string(text: str) -> Score` + +### `to_lilypond(score: Score, path: str | None=None, *, language: str | None=None, relative: bool | None=None, version: LilyPondVersion | str | None=None) -> str` + +*relative*: ``True`` for ``\relative`` entry, ``False`` for absolute, +``None`` for the score's own. + +### `to_lilypond_music(doc: MusicDocument, path: str | None=None, *, version: LilyPondVersion | str | None=None) -> str` + +### `to_musicxml(score: Score, path: str | None=None) -> str` + +### `to_mxl_bytes(score: Score) -> bytes` + +Serialize a score to compressed MusicXML (a ZIP archive). + +### `to_abc(score: Score, path: str | None=None) -> str` + +### `to_humdrum(score: Score, path: str | None=None) -> str` + + +## LilyPond \include flattening + +### `flatten(input: str, output: str | None=None, *, include_paths: list[str] | None=None, add_markers: bool=True) -> str` + +### `flatten_string(text: str, *, base_dir: str | None=None, include_paths: list[str] | None=None, add_markers: bool=True) -> str` + +Expand the ``\include`` directives of LilyPond text (relative ones +against *base_dir*, then *include_paths*), found on the parse tree. +ParseError for a missing file (LilyPond's own, like ``english.ly``, stay +as they are), a circular include or an expansion past the bounds. + + +## Transforms + +### `transpose(music: _M, semitones: int) -> _M` + +### `transpose_interval(music: _M, interval: str) -> _M` + +### `transpose_to_key(music: _M, key: str) -> _M` + +### `change_language(music: _M, language: str) -> _M` + +### `invert(music: _M, *, step: str='C', alter: int=0, octave: int=4) -> _M` + +### `retrograde(music: _M) -> _M` + + +## ML representations (Epic D) + +### `to_note_array(doc: MusicDocument, resolution: int=480) -> npt.NDArray[np.int32]` + +Encode a document as a ``(N, 4)`` array: (onset, duration, pitch, velocity). + +### `from_note_array(array: npt.NDArray[np.int32], resolution: int=480) -> MusicDocument` + +Decode a ``(N, 4)`` note array back into a document. + +### `to_event_sequence(doc: MusicDocument, resolution: int=480, max_time_shift: int=100, velocity_bins: int=32, encode_velocity: bool=True) -> npt.NDArray[np.int64]` + +Encode a document as a 1-D event-code sequence (Performance-RNN style). + +### `from_event_sequence(array: npt.NDArray[np.int64], resolution: int=480, max_time_shift: int=100, velocity_bins: int=32, encode_velocity: bool=True) -> MusicDocument` + +Decode an event-code sequence back into a document. + +### `to_piano_roll(doc: MusicDocument, resolution: int=480, encode_velocity: bool=True) -> npt.NDArray[np.uint8]` + +Encode a document as a ``(T, 128)`` piano-roll matrix. + +### `from_piano_roll(array: npt.NDArray[np.uint8], resolution: int=480, encode_velocity: bool=True) -> MusicDocument` + +Decode a ``(T, 128)`` piano-roll matrix back into a document. + +### `compute_metrics(doc: MusicDocument, resolution: int=480, measure_resolution: int | None=None) -> dict[str, Any]` + +Compute objective evaluation metrics (NaN where undefined). + +Keys: n_pitches_used, n_pitch_classes_used, pitch_range, +pitch_class_histogram, pitch_entropy, pitch_class_entropy, polyphony, +polyphony_rate, empty_beat_rate, scale_consistency, groove_consistency. + + +## MIDI + +### `from_midi(path: str, *, quantize: int | None=None, swing: bool | None=None) -> Score` + +Parse a Standard MIDI File. ``quantize`` (4, 8, 16 or 32) is the +shortest plain note value a played file is snapped to; ``swing`` reads +swung eighths as straight ones marked "Swing" (``True``), never +(``False``), or when a played file swings (``None``). + +### `from_midi_bytes(data: bytes, *, quantize: int | None=None, swing: bool | None=None) -> Score` + +Parse a Standard MIDI File from in-memory bytes. + +### `to_midi(score: Score, path: str, *, unfold_repeats: bool=True) -> None` + +Write a score to a Standard MIDI File, playing repeats out unless +``unfold_repeats=False``. + +### `to_midi_bytes(score: Score, *, unfold_repeats: bool=True) -> bytes` + +Serialize a score to Standard MIDI File bytes. + + +## Datasets (`lytk.datasets`) + +### class `Dataset` + +Abstract base: an indexable collection of :class:`MusicDocument` objects. + +Subclasses implement :meth:`__len__` and :meth:`__getitem__`. Everything +else (representation conversion, metrics, splits, torch/tf adapters) is +built on top of those two. + +``on_error`` says what iterating does with an item that cannot be read: +``"raise"`` (the default) raises, ``"skip"`` leaves it out and records it +in :attr:`errors`, ``"warn"`` does the same and warns. Iteration covers +``for doc in ds``, :meth:`iter_representation`, :meth:`to_representation`, +:meth:`metrics` and the TensorFlow adapters; indexing (``ds[i]``, the +torch adapters) always raises for such an item. + +#### `Dataset.ids: list[str]` + +An identifier per item, in order (here the index as a string). + +#### `Dataset.errors: dict[str, str]` + +The items iteration skipped: id → ``"ErrorType: message"``. + +#### `Dataset.iter_representation(representation: str, **kwargs: Any) -> Iterator[np.ndarray]` + +Lazily convert each item to ``representation``, one array at a time. + +``representation`` is one of ``"note_array"``, ``"event_sequence"``, +``"piano_roll"``; ``**kwargs`` are forwarded to the converter. Unlike +:meth:`to_representation`, this never materialises the whole dataset in +memory, so it scales to large folders. When a ``cache_dir`` is set the +per-item cache is consulted as each item is produced. + +#### `Dataset.to_representation(representation: str, **kwargs: Any) -> list[np.ndarray]` + +Eagerly convert every item to ``representation`` (one array per item +that can be read). + +This materialises the whole dataset; use :meth:`iter_representation` +for a memory-bounded stream over large corpora. + +#### `Dataset.to_note_arrays(resolution: int=480) -> list[np.ndarray]` + +#### `Dataset.to_event_sequences(**kwargs: Any) -> list[np.ndarray]` + +#### `Dataset.to_pianorolls(resolution: int=480, encode_velocity: bool=True) -> list[np.ndarray]` + +#### `Dataset.metrics(resolution: int=480, measure_resolution: int | None=None) -> list[dict[str, Any]]` + +Compute objective metrics for every item that can be read. + +#### `Dataset.split(ratios: Sequence[float]=(0.8, 0.1, 0.1), seed: int=0, *, groups: Any=None) -> tuple['Subset', ...]` + +Deterministically partition into subsets by the given ratios. + +Returns one :class:`Subset` per ratio (e.g. train/val/test). With +``groups`` (one key per item, or a function of the item's id), items of +a group land in the same subset: the groups are shuffled and each goes +to the subset furthest below its share of the items. + +#### `Dataset.to_pytorch_dataset(representation: str='note_array', *, return_ids: bool=False, **kwargs: Any)` + +Return a ``torch.utils.data.Dataset`` yielding tensors of the +chosen representation, or ``(tensor, id)`` pairs with ``return_ids``. +Requires PyTorch. + +Conversion is lazy: each item is converted (and cached, if a +``cache_dir`` is set) only when it is indexed, so constructing the +dataset does not materialise the whole corpus in memory. + +#### `Dataset.to_tensorflow_dataset(representation: str='note_array', *, return_ids: bool=False, **kwargs: Any)` + +Return a ``tf.data.Dataset`` of the chosen representation, or of +``(tensor, id)`` pairs with ``return_ids``. Requires TensorFlow. + +The dataset streams items through a generator, converting (and +caching, if a ``cache_dir`` is set) one item at a time rather than +building the whole corpus up front. Items that cannot be read are +skipped or raise as ``on_error`` says. + +#### `Dataset.to_pytorch_dataloader(representation: str='note_array', *, batch_size: int=1, shuffle: bool=False, pad_value: float=0.0, return_ids: bool=False, representation_kwargs: dict[str, Any] | None=None, **loader_kwargs: Any)` + +Return a ready-to-train ``torch.utils.data.DataLoader``. + +Like :meth:`to_pytorch_dataset` but batched, with :func:`pad_collate` +wired in so that ``batch_size > 1`` works on ragged scores. Each batch +is a ``(padded, lengths)`` tuple, ``(padded, lengths, ids)`` with +``return_ids``. + +Unlike the other methods on this class, ``**kwargs`` here go to the +``DataLoader`` (``num_workers``, ``pin_memory``, ``drop_last``, …); +arguments for the representation converter go in +``representation_kwargs``. Pass your own ``collate_fn`` to override the +padding behaviour. + +```python +train, val, test = FolderDataset("corpus/").split() +loader = train.to_pytorch_dataloader( + "event_sequence", batch_size=32, shuffle=True, num_workers=4 +) +for events, lengths in loader: + ... +``` + +#### `Dataset.to_tensorflow_dataloader(representation: str='note_array', *, batch_size: int=1, shuffle: bool=False, pad_value: float=0.0, return_ids: bool=False, representation_kwargs: dict[str, Any] | None=None)` + +Return a batched ``tf.data.Dataset`` of ``(padded, lengths)`` tuples, +``(padded, lengths, ids)`` with ``return_ids``. + +The TensorFlow counterpart of :meth:`to_pytorch_dataloader`: pads each +batch along the ragged first axis with ``padded_batch`` and carries the +true lengths alongside, for the same reason as :func:`pad_collate` +(``0`` is a valid value, so padding is not self-identifying). + +```python +loader = train.to_tensorflow_dataloader("piano_roll", batch_size=16) +for rolls, lengths in loader: + ... +``` + +### class `FolderDataset(Dataset)` + +A dataset over the supported music files found in a directory. + +Documents are loaded lazily on access; an item's id is its path relative to +``root``. ``movements="all"`` makes each movement of a file (each +``\score`` of a LilyPond file, each tune of an ABC file) an item. +``language``, ``include_paths`` and ``strict`` are passed to the LilyPond +reader, ``quantize`` to the MIDI one. ``on_error`` is described in +:class:`Dataset`. When ``cache_dir`` is given, converted representations +are cached to ``.npy`` files keyed by the file's content, how it is read, +the lytk version and the conversion parameters (the content of included +files is not part of the key). + +#### `FolderDataset(root: str | Path, recursive: bool=True, extensions: Sequence[str] | None=None, cache_dir: str | Path | None=None, *, on_error: str='raise', movements: str='first', language: str | None=None, include_paths: Sequence[str | Path] | None=None, strict: bool=False, quantize: int | None=None)` + +#### `FolderDataset.ids: list[str]` + +The source's id per item; with ``movements="all"``, followed by +``#`` and the movement's number, from 1. + +#### `FolderDataset.filenames: list[str]` + +### class `RecordsDataset(Dataset)` + +A dataset over records (dicts) holding music as text: each record's +``text_field`` in ``format`` (``"lilypond"``, ``"musicxml"``, ``"abc"`` or +``"humdrum"``), identified by its ``id_field``. + +Items keep their record (:meth:`record`), and their id is the record's +(followed by ``#`` and the movement's number with ``movements="all"``). +With ``split_field``, the records carry their own splits: :meth:`split` +returns them instead of re-shuffling by ratio. The other options are as in +:class:`FolderDataset`; the cache is keyed by the record's text. + +#### `RecordsDataset(records: Iterable[dict[str, Any]], *, text_field: str='text', id_field: str='id', format: str='lilypond', split_field: str | None=None, cache_dir: str | Path | None=None, on_error: str='raise', movements: str='first', language: str | None=None, include_paths: Sequence[str | Path] | None=None, strict: bool=False)` + +#### `RecordsDataset.ids: list[str]` + +The source's id per item; with ``movements="all"``, followed by +``#`` and the movement's number, from 1. + +#### class method `RecordsDataset.from_records(records: Iterable[dict[str, Any]], **kwargs: Any) -> 'RecordsDataset'` + +A dataset over an iterable of records (keywords as the class). + +#### class method `RecordsDataset.from_jsonl(path: str | Path, **kwargs: Any) -> 'RecordsDataset'` + +A dataset over a JSON Lines file, one record per line (keywords as +the class). + +#### `RecordsDataset.record(index: int) -> dict[str, Any]` + +The record item ``index`` comes from. + +#### `RecordsDataset.split(ratios: Sequence[float] | None=None, seed: int=0, *, groups: Any=None, field: str | None=None)` + +The records' own splits, or a ratio split. + +With ``field`` (or ``split_field`` given to the dataset, when no +``ratios`` are), returns ``{value: Subset}`` for each value of that +field, in order of first appearance: a curated, decontaminated split +stays as it is. Otherwise splits by ``ratios`` (default +``(0.8, 0.1, 0.1)``) as :meth:`Dataset.split` does; ``groups`` may name +a field whose equal values stay together. + +### class `Subset(Dataset)` + +A view of a parent dataset restricted to a list of indices. Its items, +ids, cache and errors are the parent's. + +#### `Subset(dataset: Dataset, indices: Sequence[int])` + +#### `Subset.ids: list[str]` + +#### `Subset.on_error: str` + +#### `Subset.errors: dict[str, str]` + +### `load_document(path: str | Path, *, language: str | None=None, include_paths: Sequence[str] | None=None, strict: bool=False, quantize: int | None=None)` + +Load any supported music file as a :class:`MusicDocument` (its first +movement). + +LilyPond is parsed directly into the Layer-1 Music tree; every other format +is parsed into a Score and lifted. ``language``, ``include_paths`` and +``strict`` are passed to the LilyPond reader, ``quantize`` to the MIDI one. + +### `pad_collate(batch: Sequence[Any], pad_value: float=0.0)` + +Collate variable-length representation tensors into one padded batch. + +Every representation is ragged along its first axis — note arrays are +``(n_notes, 4)``, event sequences ``(n_events,)``, piano rolls +``(n_frames, 128)`` — and the number of notes/events/frames differs per +score, so ``torch``'s default collate (which stacks) raises +``RuntimeError: stack expects each tensor to be equal size``. This pads the +first axis to the longest item in the batch. + +Returns ``(padded, lengths)``, or ``(padded, lengths, ids)`` when the items +are ``(tensor, id)`` pairs (a dataset made with ``return_ids=True``). The +lengths are returned rather than left to be inferred from ``pad_value``, +because the padding value is not reserved: ``0`` is a legitimate event +code, pitch and velocity, so trailing zeros are genuinely ambiguous. Use +them to build a mask or to ``pack_padded_sequence``. + +Suitable as a ``collate_fn`` for any ``torch.utils.data.DataLoader``: + +```python +from functools import partial +DataLoader(ds, batch_size=8, collate_fn=partial(pad_collate, pad_value=-1)) +``` + +### Constants + +- `SUPPORTED_EXTENSIONS`: `['.abc', '.ily', '.kern', '.krn', '.ly', '.mid', '.midi', '.musicxml', '.mxl', '.xml']` +- `TEXT_FORMATS`: `('lilypond', 'musicxml', 'abc', 'humdrum')` diff --git a/docs/roadmap.md b/docs/roadmap.md index b1da735..aa50a1d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,7 +2,35 @@ Items are grouped by status. Completed items are kept for reference. -## Latest status (2026-09-27, Epic J: J6, the hunt, J7) +## Latest status (2026-09-28, Epics K and L) + +**Epics K and L are done and released as v0.4.0.** Epic L (datasets) ships +in it too, as the owner chose. Datasets read records as well as folders +(`RecordsDataset`, with the records' own splits and ids), skip what they +cannot read when asked (`on_error`), make an item of each movement when +asked, cache by content, and the torch and TensorFlow adapters can return +ids; `docs/python-api.md` is generated from the stubs. Since K: embedded +Scheme is tokenized as Scheme, and `from_lilypond_movements_string` reads +every movement of text. Test counts: 1,158 Rust + 262 Python. + +--- + +## Previous status (2026-09-28, Epic K) + +**Epic K is done.** LilyPond source can be inspected +and edited without reading the music: its `\version` (K1: +`lytk.LilyPondVersion`, `lilypond_version`, `set_`/`strip_lilypond_version`, +`version=` on the writers, the `invalid-version` error), its tokens (K3: +`lytk.tokenize`, `strip_comments`) and its statistics (K4: `lytk.info`, +`source_stats`). Includes (K2) are found on the tree, `flatten_string` +flattens text, and every reader follows includes when given +`include_paths=`, with diagnostics pointing into the caller's text. +`from_lilypond_music_movements` (K5) completes the movement readers. Test +counts: 1,157 Rust + 241 Python. Details in the devlog. + +--- + +## Previous status (2026-09-27, Epic J: J6, the hunt, J7) **Epic J is done and released as v0.3.0.** The version has one source and the Python tests run on 3.10–3.13 (J6). The hunt's last five areas found two @@ -697,29 +725,31 @@ what it drops. Principles, as in Epic I: Order: J0 → J1 + J2 → J3 → J4 + J5 → J6 → J7. -### Epic K — 0.4.0: source-level API +### Epic K — 0.4.0: source-level API (✅ released in v0.4.0, 2026-09-28) | Task | Description | Size | Status | |------|-------------|------|--------| -| K1 | **`\version`.** Keep it in `ScoreMetadata` and expose it as `Score.lilypond_version`. `lytk.lilypond_version(text)` reads it from the tree (a commented-out one does not count). `set_lilypond_version(text, v)` and `strip_lilypond_version(text)` edit by span. A `LilyPondVersion` type compares numerically (`2.24` == `2.24.0`). `to_lilypond(…, version=)` and `to_lilypond_music(…, version=)` expose `IrToLyAdapter::with_version`: Python always gets `\version "2.24.0"` today (`python.rs:398, 421`). No convert-ly wrapper | S | ⬜ | -| K2 | **Includes.** Bind `flatten_string(text, *, base_dir=None, include_paths=(), add_markers=True)` (Rust `flatten_str` exists, `ly_flatten.rs:100-107`). Add `include_paths=` on the readers: given, they flatten first and report unresolved includes as `ignored-include`; not given, includes stay unfollowed, as SECURITY.md should say. Find includes on the tree instead of line by line: this catches includes not at the start of a line and fixes the one-line `%{ %}` that leaves flatten "inside a comment" and the false `MultipleHeaders` for a top-level plus a per-score `\header` | M | ⬜ | -| K3 | **Tokens.** `lytk.tokenize(text) -> list[Token]` (kind: comment, string, scheme, command, symbol, number, fraction, punctuation; text; byte span; line; column) from the tree's leaves; `lytk.strip_comments(text)`. Scheme can be sub-tokenized with the compiled but unused `LANGUAGE_LILYPOND_SCHEME` | S | ⬜ | -| K4 | **Statistics.** `lytk.info(score) -> dict` becomes the library home of `lytk info --json` (moved from `cli.py:484-529`; the CLI calls it), plus voices, total bars, duration in quarters, lyrics, chord symbols and grace notes. `lytk.source_stats(text)` from tokens: comments, Scheme expressions, bytes, tokens | S | ⬜ | -| K5 | **Movements.** Single-score readers report what they drop (`dropped-music`, from J3); add `from_lilypond_music_movements` for Layer-1 users (the datasets) | S | ⬜ | +| K1 | **`\version`.** Keep it in `ScoreMetadata` and expose it as `Score.lilypond_version`. `lytk.lilypond_version(text)` reads it from the tree (a commented-out one does not count). `set_lilypond_version(text, v)` and `strip_lilypond_version(text)` edit by span. A `LilyPondVersion` type compares numerically (`2.24` == `2.24.0`). `to_lilypond(…, version=)` and `to_lilypond_music(…, version=)` expose `IrToLyAdapter::with_version`: Python always gets `\version "2.24.0"` today (`python.rs:398, 421`). No convert-ly wrapper | S | ✅ 2026-09-28: `ly_to_ir/version.rs`, one scan of the tree for the reader, the syntax check and the three text functions; LilyPond 2.24's rules (`parse-lily-version`: `x.y` only for an even `y`, a free fourth part); a new `invalid-version` error, as LilyPond's lexer rejects both a bad string and a missing one. The writers keep writing 2.24.0 unless told: their syntax is 2.24's, whatever the source's version | +| K2 | **Includes.** Bind `flatten_string(text, *, base_dir=None, include_paths=(), add_markers=True)` (Rust `flatten_str` exists, `ly_flatten.rs:100-107`). Add `include_paths=` on the readers: given, they flatten first and report unresolved includes as `ignored-include`; not given, includes stay unfollowed, as SECURITY.md should say. Find includes on the tree instead of line by line: this catches includes not at the start of a line and fixes the one-line `%{ %}` that leaves flatten "inside a comment" and the false `MultipleHeaders` for a top-level plus a per-score `\header` | M | ✅ 2026-09-28: `include_paths=` on every reader and on `check_lilypond`; flatten records where each piece of its output comes from, so diagnostics point into the caller's text (in an included file: at its `\include`, naming it). `MultipleHeaders` is gone altogether, since LilyPond merges top-level headers (`get_header`, `lily/lily-parser.cc`). Also fixed: flatten kept only the last `\language`, which changes the pitch names before it; an include of LilyPond's own files (`english.ly`, generated list of its `ly/` directory) is kept instead of failing | +| K3 | **Tokens.** `lytk.tokenize(text) -> list[Token]` (kind: comment, string, scheme, command, symbol, number, fraction, punctuation; text; byte span; line; column) from the tree's leaves; `lytk.strip_comments(text)`. Scheme can be sub-tokenized with the compiled but unused `LANGUAGE_LILYPOND_SCHEME` | S | ✅ 2026-09-28: `ly_to_ir/tokens.rs`; strings, Scheme and quoted identifiers whole, `\<`-style marks are commands, and an `error` kind for text the grammar cannot tokenize, so tokens lose nothing but whitespace (checked on the fixtures, broken input, and the 2,706 files of the corpus board). Offsets are characters in Python. Embedded Scheme is tokenized as Scheme from the main grammar's own `scheme_*` nodes (no second grammar needed), each token marked `scheme`; LilyPond inside `#{ … #}` as LilyPond | +| K4 | **Statistics.** `lytk.info(score) -> dict` becomes the library home of `lytk info --json` (moved from `cli.py:484-529`; the CLI calls it), plus voices, total bars, duration in quarters, lyrics, chord symbols and grace notes. `lytk.source_stats(text)` from tokens: comments, Scheme expressions, bytes, tokens | S | ✅ 2026-09-28: counted in Rust (`ir/stats.rs`) instead of through the navigation objects, so a corpus pays microseconds a score; the CLI's keys stay, the new ones are added; `source_stats` also gives lines and error tokens | +| K5 | **Movements.** Single-score readers report what they drop (`dropped-music`, from J3); add `from_lilypond_music_movements` for Layer-1 users (the datasets) | S | ✅ 2026-09-28: the first half came with J3 and J7 | + +### Epic L — datasets for curated corpora (✅ released with K in v0.4.0, 2026-09-28) -### Epic L — 0.5.0: datasets for curated corpora +Planned for 0.5.0; the owner chose to ship it in 0.4.0 together with Epic K. | Task | Description | Size | Status | |------|-------------|------|--------| -| L1 | **`FolderDataset` robustness.** `on_error="raise" \| "skip" \| "warn"` with `dataset.errors` (today the first bad file ends every iteration). Cache keyed by file content + lytk version + normalized representation kwargs, written atomically (today keyed by path, never invalidated; `to_note_arrays()` and `iter_representation("note_array")` write different files). `Subset` delegates `_convert_item` to its parent: today it skips the cache, so the README's `split()` → dataloader example never uses it. `movements="first" \| "all"`. Pass-through of `language`, `include_paths`, `strict` and MIDI `quantize` | M | ⬜ | -| L2 | **Records dataset.** `RecordsDataset.from_jsonl(path, *, text_field="text", id_field="id", format="lilypond", split_field=None)` and `from_records(iterable)`. Items keep their id and metadata; `split(field=…)` returns the records' own splits (a deterministic, decontaminated split must not be re-shuffled by ratio). `Dataset.split(…, groups=)` for group-aware ratio splits | M | ⬜ | -| L3 | **Identity.** `FolderDataset.ids` (paths relative to the root); `return_ids=True` on the torch and tf adapters | S | ⬜ | -| L4 | **Python API reference.** `docs/python-api.md` generated from or checked against the stubs: lytk has none today (README + `_core.pyi`). It states that `from_lilypond_string` and `from_lilypond_music_string` share one parse (the Music tree is lifted from the Score) | S | ⬜ | +| L1 | **`FolderDataset` robustness.** `on_error="raise" \| "skip" \| "warn"` with `dataset.errors` (today the first bad file ends every iteration). Cache keyed by file content + lytk version + normalized representation kwargs, written atomically (today keyed by path, never invalidated; `to_note_arrays()` and `iter_representation("note_array")` write different files). `Subset` delegates `_convert_item` to its parent: today it skips the cache, so the README's `split()` → dataloader example never uses it. `movements="first" \| "all"`. Pass-through of `language`, `include_paths`, `strict` and MIDI `quantize` | M | ✅ 2026-09-28: iteration skips or raises as `on_error` says, indexing always raises; the cache key hashes the content with the reading options, lytk's version and the converter's arguments with their defaults filled in (content of included files excepted); `movements="all"` ids are `path#1`, `path#2`, … | +| L2 | **Records dataset.** `RecordsDataset.from_jsonl(path, *, text_field="text", id_field="id", format="lilypond", split_field=None)` and `from_records(iterable)`. Items keep their id and metadata; `split(field=…)` returns the records' own splits (a deterministic, decontaminated split must not be re-shuffled by ratio). `Dataset.split(…, groups=)` for group-aware ratio splits | M | ✅ 2026-09-28: with `split_field`, `split()` returns `{value: Subset}` as recorded (ratios are then a deliberate re-split); `groups` is one key per item, a function of the id, or for records a field name; each shuffled group goes to the subset furthest below its share. `from_abc_tunes_string` added for `movements="all"` on ABC records | +| L3 | **Identity.** `FolderDataset.ids` (paths relative to the root); `return_ids=True` on the torch and tf adapters | S | ✅ 2026-09-28: `ids` on every dataset (records: their ids); `return_ids` on the two datasets and the two data loaders, whose batches gain the ids | +| L4 | **Python API reference.** `docs/python-api.md` generated from or checked against the stubs: lytk has none today (README + `_core.pyi`). It states that `from_lilypond_string` and `from_lilypond_music_string` share one parse (the Music tree is lifted from the Score) | S | ✅ 2026-09-28: `scripts/python_api.py` writes it from the stubs and the datasets module, in the sections of `lytk.__all__`; tests fail when it is stale or when a public name of the extension has no stub | -### After 0.5.0 +### After 0.4.0 The pre-1.0 queue resumes: **P5** (enumerated notation types, the one -epic that breaks the Python API) as 0.6.0, then P3 → P4 → P6 → P8 → 1.0.0. +epic that breaks the Python API) as 0.5.0, then P3 → P4 → P6 → P8 → 1.0.0. Decision for the owner: J–L go before P5 because they unblock lilycorpus and are backward compatible, whereas the 0.1.0 plan put P5 first. P3's import-options object should absorb J3/K2's keywords one to one: @@ -930,9 +960,8 @@ notation-exported MIDI (IC). | **12** | **0.1.0** | **Preview release before the remaining P-epics; then P5 → P3 → P4 → P6 → P8 → P9** | | **13** | **I → 0.2.0** | **MIDI and ABC conversion repair (Epic I, with P9 performed MIDI): tagged `v0.2.0` 2026-09-27** | | **14** | **J → 0.3.0** | **Trustworthy LilyPond reading: no panics, diagnostics + strict mode, whole strings and headers, pitch-language files: tagged `v0.3.0` 2026-09-27** | -| 15 | K → 0.4.0 | Source-level API: `\version`, includes from strings, tokens, statistics, movements | -| 16 | L → 0.5.0 | Datasets for curated corpora: robust `FolderDataset`, records dataset, ids, Python API reference | -| 17 | P5 → 0.6.0, then P3 → P4 → P6 → P8 → 1.0.0 | Pre-1.0 queue resumes (P5 breaks the Python API) | +| **15** | **K + L → 0.4.0** | **Source-level API (`\version`, includes, tokens, statistics, movements) and datasets for curated corpora (records dataset, errors, ids, Python API reference): tagged `v0.4.0` 2026-09-28** | +| 16 | P5 → 0.5.0, then P3 → P4 → P6 → P8 → 1.0.0 | Pre-1.0 queue resumes (P5 breaks the Python API) | ## Key Decisions diff --git a/scripts/ly_builtins.py b/scripts/ly_builtins.py index 8c7c96a..51a622e 100644 --- a/scripts/ly_builtins.py +++ b/scripts/ly_builtins.py @@ -87,15 +87,21 @@ def main(repo: str, tag: str) -> None: write_drums(repo, tag) names = sorted(names) rows = "\n".join(f' "{n.replace(chr(92), chr(92) * 2)}",' for n in names) + # The files an `\include` finds in LilyPond's own `ly/` directory. + shipped = sorted(p.removeprefix("ly/") for p in files(repo, tag, "ly", ".ly")) + shipped_rows = "\n".join(f' "{n}",' for n in shipped) with open("src/adapters/ly_to_ir/builtins.rs", "w") as out: out.write( f"//! The commands LilyPond {tag.lstrip('v')} defines, without their backslash:\n" "//! lexer keywords, the identifiers of its `ly/` init files and its markup\n" - f"//! commands ({len(names)}). Generated: `python3 scripts/ly_builtins.py lilypond {tag}`.\n\n" + f"//! commands ({len(names)}); and the files of its `ly/` directory ({len(shipped)}).\n" + f"//! Generated: `python3 scripts/ly_builtins.py lilypond {tag}`.\n\n" "/// Sorted, for binary search.\n" - f"pub(super) const BUILTINS: &[&str] = &[\n{rows}\n];\n" + f"pub(super) const BUILTINS: &[&str] = &[\n{rows}\n];\n\n" + "/// Files LilyPond itself provides to `\\include`, sorted.\n" + f"pub(crate) const LILYPOND_FILES: &[&str] = &[\n{shipped_rows}\n];\n" ) - print(len(names), "commands") + print(len(names), "commands,", len(shipped), "files") if __name__ == "__main__": diff --git a/scripts/python_api.py b/scripts/python_api.py new file mode 100644 index 0000000..ede7071 --- /dev/null +++ b/scripts/python_api.py @@ -0,0 +1,177 @@ +"""Write docs/python-api.md, the Python API reference, from the stubs. + +Run `python scripts/python_api.py` after changing the stubs +(`src/lytk/_core.pyi`), `lytk.__all__` or the datasets module; +`tests/test_bindings.py` fails when the reference is out of date. Sections +follow the comments in `lytk.__all__`; signatures and docstrings come from the +stubs and from `src/lytk/datasets/base.py`, read with `ast`. +""" + +from __future__ import annotations + +import ast +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OUT = ROOT / "docs" / "python-api.md" + +INTRO = """\ +# Python API reference + +*Generated from `src/lytk/_core.pyi` and `src/lytk/datasets/` by +`python scripts/python_api.py`; a test fails when this file is out of date.* + +`import lytk` gives everything below; `lytk.__version__` is the version. + +- **Errors.** Readers (`from_*`, `Score.from_json`/`from_dict`, + `MusicDocument.from_json`, `flatten`) raise `lytk.ParseError`, a + `ValueError`, when the input cannot be read, and `OSError` when a file + cannot be opened. `strict=True` LilyPond readers raise + `lytk.LilyPondSyntaxError`, a `ParseError`, when the reading has an error. + Writers (`to_*`) raise `ValueError` when a score cannot be written. Any + function may raise `lytk.InternalError`, a bug in lytk. +- **Positions** (`Diagnostic`, `HeaderField`, `Token`) are character + offsets: `text[d.start:d.end]` is the span. +- **One LilyPond parse.** `from_lilypond_string` and + `from_lilypond_music_string` (and their file twins) run the same parse, + walk and assembly: the Music tree of `MusicDocument` is lifted from the + `Score`. Whatever one accepts, the other accepts too. +""" + + +def all_sections(init: str) -> list[tuple[str, list[str]]]: + """(title, names) in the order of `lytk.__all__`, one per comment.""" + block = re.search(r"^__all__ = \[\n(.*?)^\]", init, re.S | re.M) + if block is None: + raise SystemExit("no __all__ list in lytk/__init__.py") + sections: list[tuple[str, list[str]]] = [] + for line in block.group(1).splitlines(): + line = line.strip() + if line.startswith("#"): + sections.append((line.lstrip("# "), [])) + elif m := re.fullmatch(r'"(\w+)",', line): + if not sections: + raise SystemExit(f"{m.group(1)} in __all__ before any section comment") + sections[-1][1].append(m.group(1)) + extra = re.search(r"^__all__ \+= \[(.*?)\]", init, re.M) + if extra: + sections.append(("MIDI", re.findall(r'"(\w+)"', extra.group(1)))) + return sections + + +def definitions(source: str) -> dict[str, ast.stmt]: + return { + node.name: node + for node in ast.parse(source).body + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) + } + + +def signature(fn: ast.FunctionDef, name: str) -> str: + args = fn.args + if args.args and args.args[0].arg in ("self", "cls"): + args = ast.arguments( + posonlyargs=args.posonlyargs, + args=args.args[1:], + vararg=args.vararg, + kwonlyargs=args.kwonlyargs, + kw_defaults=args.kw_defaults, + kwarg=args.kwarg, + defaults=args.defaults, + ) + returns = f" -> {ast.unparse(fn.returns)}" if fn.returns else "" + return f"{name}({ast.unparse(args)}){returns}" + + +def doc(node: ast.AST) -> str: + text = ast.get_docstring(node) or "" + return f"{text}\n\n" if text else "" + + +def decorators(fn: ast.FunctionDef) -> set[str]: + return {ast.unparse(d) for d in fn.decorator_list} + + +def render_class(cls: ast.ClassDef, module: dict[str, ast.stmt]) -> str: + # A private base of the same module is shown as the public class it + # derives from, and its public members as the class's own. + members: dict[str, ast.FunctionDef] = {} + shown_bases = [] + for base in cls.bases: + name = ast.unparse(base) + private = module.get(name) if name.startswith("_") else None + if isinstance(private, ast.ClassDef): + shown_bases.extend(ast.unparse(b) for b in private.bases) + members.update({m.name: m for m in private.body if isinstance(m, ast.FunctionDef)}) + else: + shown_bases.append(name) + members.update({m.name: m for m in cls.body if isinstance(m, ast.FunctionDef)}) + bases = ", ".join(shown_bases) + out = f"### class `{cls.name}{f'({bases})' if bases else ''}`\n\n{doc(cls)}" + # The constructor first, then the members in the order they are defined. + for member in sorted(members.values(), key=lambda m: m.name != "__init__"): + name = member.name + if name == "__init__": + member.returns = None + out += f"#### `{signature(member, cls.name)}`\n\n{doc(member)}" + elif name.startswith("_"): + continue + elif "property" in decorators(member): + kind = f": {ast.unparse(member.returns)}" if member.returns else "" + out += f"#### `{cls.name}.{name}{kind}`\n\n{doc(member)}" + elif not any(d.endswith(".setter") for d in decorators(member)): + prefix = "static " if "staticmethod" in decorators(member) else "" + prefix = "class method " if "classmethod" in decorators(member) else prefix + out += f"#### {prefix}`{signature(member, f'{cls.name}.{name}')}`\n\n{doc(member)}" + return out + + +def render_entry(node: ast.stmt, module: dict[str, ast.stmt]) -> str: + if isinstance(node, ast.ClassDef): + return render_class(node, module) + assert isinstance(node, ast.FunctionDef) + return f"### `{signature(node, node.name)}`\n\n{doc(node)}" + + +def render() -> str: + stubs = definitions((ROOT / "src/lytk/_core.pyi").read_text()) + sections = all_sections((ROOT / "src/lytk/__init__.py").read_text()) + out = INTRO + missing = [] + for title, names in sections: + out += f"\n## {title}\n\n" + for name in names: + if name not in stubs: + missing.append(name) + continue + out += render_entry(stubs[name], stubs) + if missing: + raise SystemExit(f"no stub for {', '.join(missing)} in src/lytk/_core.pyi") + # lytk.datasets: its own module, documented from the source. + datasets = definitions((ROOT / "src/lytk/datasets/base.py").read_text()) + exported = re.findall( + r'"(\w+)"', re.search(r"__all__ = \[(.*?)\]", (ROOT / "src/lytk/datasets/__init__.py").read_text(), re.S).group(1) + ) + out += "\n## Datasets (`lytk.datasets`)\n\n" + for name in exported: + if name in datasets: + out += render_entry(datasets[name], datasets) + constants = [n for n in exported if n not in datasets] + if constants: + out += "### Constants\n\n" + sys.path.insert(0, str(ROOT / "src")) + import lytk.datasets as lds + + for name in constants: + value = getattr(lds, name) + shown = sorted(value) if isinstance(value, (set, frozenset)) else value + out += f"- `{name}`: `{shown!r}`\n" + out += "\n" + return out.rstrip() + "\n" + + +if __name__ == "__main__": + OUT.write_text(render()) + print(f"wrote {OUT.relative_to(ROOT)}") diff --git a/src/adapters/ly_flatten.rs b/src/adapters/ly_flatten.rs index 9066eff..b510bea 100644 --- a/src/adapters/ly_flatten.rs +++ b/src/adapters/ly_flatten.rs @@ -18,8 +18,15 @@ //! | Command | Behaviour | //! |--------------|------------------------------------------------| //! | `\version` | Last occurrence kept; earlier ones removed; warning emitted if multiple | -//! | `\language` | Last occurrence kept; earlier ones removed; warning emitted if multiple | -//! | `\header` | Multiple blocks are an **error** | +//! +//! `\language` lines all stay (each applies from where it stands), and so do +//! several `\header` blocks (LilyPond merges them). +//! +//! # Finding includes +//! Includes are found on the syntax tree: anywhere in a line, never in a +//! comment or a string. An include of a file not found is an error, except +//! for LilyPond's own files (`english.ly`, `gregorian.ly`), which LilyPond +//! finds in its installation, and with [`FlattenOpts::keep_missing`]. //! //! # Output markers //! When `FlattenOpts::add_markers` is `true` (the default), each included file @@ -54,6 +61,11 @@ pub struct FlattenOpts { /// Wrap each inlined file with `% === BEGIN/END INCLUDE: … ===` comment /// markers. Defaults to `true`. pub add_markers: bool, + + /// Keep an `\include` of a file not found as it is, instead of failing + /// with [`FlattenError::NotFound`]. An include of one of LilyPond's own + /// files (`english.ly`) is kept either way. Defaults to `false`. + pub keep_missing: bool, } impl Default for FlattenOpts { @@ -61,6 +73,7 @@ impl Default for FlattenOpts { Self { include_paths: Vec::new(), add_markers: true, + keep_missing: false, } } } @@ -81,9 +94,6 @@ pub enum FlattenError { #[error("circular include detected: {chain}")] Circular { chain: String }, - #[error("multiple \\header blocks found in flattened output")] - MultipleHeaders, - #[error("{0}; lytk refuses input this large")] TooLarge(String), } @@ -95,28 +105,59 @@ pub enum FlattenError { /// Flatten a LilyPond file at `path`, expanding all `\include` directives /// recursively and returning the combined text. /// -/// Warnings about duplicate `\version` or `\language` directives are written -/// to `stderr`. +/// Warnings about duplicate `\version` directives are written to `stderr`. pub fn flatten(path: &Path, opts: FlattenOpts) -> Result { let src = read_file(path)?; let base_dir = path.parent().unwrap_or(Path::new(".")); - flatten_str(&src, base_dir, opts) + flatten_str(&src, Some(base_dir), opts) } /// Flatten LilyPond source text `src`, resolving relative `\include` paths -/// against `base_dir`. +/// against `base_dir` (when given) and then `opts.include_paths`. /// /// This is the lower-level entry point; use [`flatten`] when working with /// files on disk. -pub fn flatten_str(src: &str, base_dir: &Path, opts: FlattenOpts) -> Result { +pub fn flatten_str( + src: &str, + base_dir: Option<&Path>, + opts: FlattenOpts, +) -> Result { + let (expanded, _) = flatten_mapped(src, base_dir, &opts)?; + normalize(expanded) +} + +/// Where the text from an offset of the flattened output on comes from: the +/// source itself, from a byte offset on; or the expansion of the `\include` +/// statement at a byte range of the source, naming the file. +#[derive(Debug, Clone)] +pub(crate) enum Origin { + Source(usize), + Include { + statement: std::ops::Range, + path: String, + }, +} + +/// The pieces of a flattened text, in order: where each starts in the output, +/// and where it comes from. A piece runs to the next one's start. +pub(crate) type SourceMap = Vec<(usize, Origin)>; + +/// `src` with its includes expanded (no normalization), and where each piece +/// of the result comes from. +pub(crate) fn flatten_mapped( + src: &str, + base_dir: Option<&Path>, + opts: &FlattenOpts, +) -> Result<(String, SourceMap), FlattenError> { let mut ctx = ExpandCtx { - opts: &opts, + opts, ancestors: Vec::new(), includes: 0, bytes: src.len(), }; - let expanded = expand(src, base_dir, &mut ctx)?; - normalize(expanded) + let mut map = SourceMap::new(); + let expanded = expand(src, base_dir, &mut ctx, Some(&mut map))?; + Ok((expanded, map)) } // --------------------------------------------------------------------------- @@ -133,116 +174,160 @@ struct ExpandCtx<'a> { bytes: usize, } -fn expand(src: &str, base_dir: &Path, ctx: &mut ExpandCtx<'_>) -> Result { - let mut out = String::with_capacity(src.len()); - let mut in_block_comment = false; - - for line in src.lines() { - let trimmed = line.trim_start(); - - // Track block comments: %{ ... %} - // LilyPond requires %{ and %} to appear as the only non-whitespace - // content on a line at the outermost level. - if trimmed.starts_with("%{") { - in_block_comment = true; - out.push_str(line); - out.push('\n'); - continue; +/// The `\include "…"` statements of `src`, from its syntax tree (so not in +/// comments or strings): each statement's byte range and its path, the text +/// between the quotes, as LilyPond's lexer takes it (no escapes). +fn include_statements(src: &str) -> Result, String)>, FlattenError> { + let tree = crate::parser::LilyPondParser::new() + .and_then(|mut p| p.parse(src)) + .map_err(|e| FlattenError::TooLarge(e.to_string()))?; + let mut out = Vec::new(); + let mut cursor = tree.walk(); + loop { + let node = cursor.node(); + if node.kind() == "escaped_word" && &src[node.byte_range()] == "\\include" { + let string = node.next_sibling().filter(|n| n.kind() == "string"); + if let Some(string) = string { + let raw = &src[string.byte_range()]; + let path = raw.trim_start_matches('"').trim_end_matches('"'); + if !path.is_empty() { + out.push((node.start_byte()..string.end_byte(), path.to_string())); + } + } } - if trimmed.starts_with("%}") { - in_block_comment = false; - out.push_str(line); - out.push('\n'); + if cursor.goto_first_child() { continue; } - - // Inside block comment or line comment — pass through unchanged. - if in_block_comment || trimmed.starts_with('%') { - out.push_str(line); - out.push('\n'); - continue; + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() { + return Ok(out); + } } + } +} - // Try to match \include "path" - if let Some(raw_path) = parse_include_directive(trimmed) { - let resolved = resolve_include(raw_path, base_dir, ctx.opts)?; - let canonical = canonicalize(&resolved)?; - - // Circular dependency check - if ctx.ancestors.contains(&canonical) { - let mut chain: Vec = ctx - .ancestors - .iter() - .map(|p| p.display().to_string()) - .collect(); - chain.push(canonical.display().to_string()); - return Err(FlattenError::Circular { - chain: chain.join(" -> "), - }); - } +/// The part of `src` an expansion replaces: the whole line when the +/// statement stands alone on it, else the statement itself. +fn replaced_span(src: &str, statement: &std::ops::Range) -> (usize, usize, bool) { + let line_start = src[..statement.start].rfind('\n').map_or(0, |i| i + 1); + let line_end = src[statement.end..] + .find('\n') + .map_or(src.len(), |i| statement.end + i + 1); + let alone = src[line_start..statement.start].trim().is_empty() + && src[statement.end..line_end].trim().is_empty(); + if alone { + (line_start, line_end, true) + } else { + (statement.start, statement.end, false) + } +} - let inc_src = read_file(&resolved)?; - ctx.includes += 1; - ctx.bytes += inc_src.len() + 2 * raw_path.len(); - if ctx.includes > MAX_INCLUDES { - return Err(FlattenError::TooLarge(format!( - "the includes expand more than {MAX_INCLUDES} times" - ))); - } - if ctx.bytes > MAX_OUTPUT_BYTES { - return Err(FlattenError::TooLarge(format!( - "the flattened source exceeds {} MiB", - MAX_OUTPUT_BYTES >> 20 - ))); - } - let inc_base = resolved.parent().unwrap_or(Path::new(".")); - - ctx.ancestors.push(canonical.clone()); - let inner = expand(&inc_src, inc_base, ctx)?; - ctx.ancestors.pop(); - - if ctx.opts.add_markers { - out.push_str(&format!("% === BEGIN INCLUDE: {} ===\n", raw_path)); - out.push_str(inner.trim_end_matches('\n')); - out.push('\n'); - out.push_str(&format!("% === END INCLUDE: {} ===\n", raw_path)); - } else { - out.push_str(&inner); +/// Whether `raw` names a file of LilyPond's own `ly/` directory, which +/// LilyPond finds when nothing else does (`english.ly`, `gregorian.ly`). +fn is_lilypond_file(raw: &str) -> bool { + super::ly_to_ir::LILYPOND_FILES.binary_search(&raw).is_ok() +} + +fn expand( + src: &str, + base_dir: Option<&Path>, + ctx: &mut ExpandCtx<'_>, + mut map: Option<&mut SourceMap>, +) -> Result { + let mut out = String::with_capacity(src.len()); + let mut at = 0; + for (statement, raw_path) in include_statements(src)? { + let resolved = match resolve_include(&raw_path, base_dir, ctx.opts) { + Ok(resolved) => resolved, + // Kept as written: LilyPond's own files, and missing files when + // asked (the readers warn about them). + Err(FlattenError::NotFound(_)) + if ctx.opts.keep_missing || is_lilypond_file(&raw_path) => + { + continue } - } else { - out.push_str(line); - out.push('\n'); + Err(e) => return Err(e), + }; + let canonical = canonicalize(&resolved)?; + + // Circular dependency check + if ctx.ancestors.contains(&canonical) { + let mut chain: Vec = ctx + .ancestors + .iter() + .map(|p| p.display().to_string()) + .collect(); + chain.push(canonical.display().to_string()); + return Err(FlattenError::Circular { + chain: chain.join(" -> "), + }); } - } - Ok(out) -} + let inc_src = read_file(&resolved)?; + ctx.includes += 1; + ctx.bytes += inc_src.len() + 2 * raw_path.len(); + if ctx.includes > MAX_INCLUDES { + return Err(FlattenError::TooLarge(format!( + "the includes expand more than {MAX_INCLUDES} times" + ))); + } + if ctx.bytes > MAX_OUTPUT_BYTES { + return Err(FlattenError::TooLarge(format!( + "the flattened source exceeds {} MiB", + MAX_OUTPUT_BYTES >> 20 + ))); + } + let inc_base = resolved.parent().unwrap_or(Path::new(".")); -/// Extract the path string from a `\include "path"` directive line. -/// Returns `None` if the line does not match. -fn parse_include_directive(line: &str) -> Option<&str> { - let rest = line.strip_prefix(r"\include")?; - let rest = rest.trim_start(); - let rest = rest.strip_prefix('"')?; - let end = rest.find('"')?; - let path = &rest[..end]; - if path.is_empty() { - None - } else { - Some(path) + ctx.ancestors.push(canonical); + let inner = expand(&inc_src, Some(inc_base), ctx, None)?; + ctx.ancestors.pop(); + + let (cut_start, cut_end, alone) = replaced_span(src, &statement); + if let Some(map) = map.as_deref_mut() { + map.push((out.len(), Origin::Source(at))); + } + out.push_str(&src[at..cut_start]); + if let Some(map) = map.as_deref_mut() { + let path = raw_path.clone(); + map.push((out.len(), Origin::Include { statement, path })); + } + if !alone && !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + if ctx.opts.add_markers { + out.push_str(&format!("% === BEGIN INCLUDE: {raw_path} ===\n")); + } + out.push_str(inner.trim_end_matches('\n')); + out.push('\n'); + if ctx.opts.add_markers { + out.push_str(&format!("% === END INCLUDE: {raw_path} ===\n")); + } + at = cut_end; + } + if let Some(map) = map { + map.push((out.len(), Origin::Source(at))); } + out.push_str(&src[at..]); + Ok(out) } -/// Resolve an include path relative to `base_dir`, with extension fallback -/// (`.ly` then `.ily`) and additional search paths. +/// Resolve an include path relative to `base_dir`, then to the include +/// paths, with extension fallback (`.ly` then `.ily`). An absolute path is +/// taken as it is. fn resolve_include( raw: &str, - base_dir: &Path, + base_dir: Option<&Path>, opts: &FlattenOpts, ) -> Result { - // Build candidate paths in order: base_dir first, then include_paths. - let search_dirs = - std::iter::once(base_dir).chain(opts.include_paths.iter().map(PathBuf::as_path)); + let search_dirs: Vec<&Path> = if Path::new(raw).is_absolute() { + vec![Path::new("")] + } else { + base_dir + .into_iter() + .chain(opts.include_paths.iter().map(PathBuf::as_path)) + .collect() + }; for dir in search_dirs { let base = dir.join(raw); @@ -310,20 +395,13 @@ fn dedup_keep_last( } } -/// Run deduplication and validation on the fully-expanded text. +/// Keep one `\version` line, the last, in the fully expanded text. +/// `\language` lines all stay: each applies from where it stands. Several +/// `\header` blocks are fine too: LilyPond merges them. fn normalize(text: String) -> Result { let lines: Vec<&str> = text.lines().collect(); - - // Phase A & B: for \version and \language, keep only the last occurrence. let mut remove: std::collections::HashSet = std::collections::HashSet::new(); dedup_keep_last(&lines, is_version_line, "\\version", &mut remove); - dedup_keep_last(&lines, is_language_line, "\\language", &mut remove); - - // Phase C: count \header blocks using brace depth tracking. - let header_count = count_header_blocks(&lines); - if header_count > 1 { - return Err(FlattenError::MultipleHeaders); - } // Reconstruct the text, skipping removed lines. let mut out = String::with_capacity(text.len()); @@ -342,52 +420,6 @@ fn is_version_line(line: &str) -> bool { t.starts_with(r#"\version ""#) } -fn is_language_line(line: &str) -> bool { - let t = line.trim_start(); - t.starts_with(r#"\language ""#) -} - -/// Count top-level `\header { ... }` blocks in the flattened output. -/// Uses brace-depth tracking; assumes `\header` is not nested inside another -/// block context (which matches typical LilyPond score structure). -fn count_header_blocks(lines: &[&str]) -> usize { - let mut count = 0usize; - let mut depth = 0i32; - let mut in_header = false; - - for line in lines { - let trimmed = line.trim_start(); - - // Skip line comments - if trimmed.starts_with('%') { - continue; - } - - if !in_header && trimmed.starts_with(r"\header") { - in_header = true; - count += 1; - } - - if in_header { - for ch in line.chars() { - match ch { - '{' => depth += 1, - '}' => { - depth -= 1; - if depth <= 0 { - depth = 0; - in_header = false; - } - } - _ => {} - } - } - } - } - - count -} - // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -413,7 +445,7 @@ mod tests { let src = r#"\version "2.24.0" { c' d' e' f' } "#; - let result = flatten_str(src, Path::new("."), FlattenOpts::default()).unwrap(); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()).unwrap(); assert!(result.contains(r#"\version "2.24.0""#)); assert!(result.contains("c' d' e' f'")); } @@ -488,6 +520,7 @@ mod tests { let opts = FlattenOpts { include_paths: vec![lib_dir.path().to_path_buf()], add_markers: false, + ..Default::default() }; let result = flatten(&dir.path().join("main.ly"), opts).unwrap(); assert!(result.contains("e'")); @@ -591,7 +624,7 @@ mod tests { let src = r#"\version "2.24.0" { c' } "#; - let result = flatten_str(src, Path::new("."), FlattenOpts::default()).unwrap(); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()).unwrap(); assert_eq!( result.matches(r#"\version"#).count(), 1, @@ -606,23 +639,23 @@ mod tests { \version "2.24.0" { d' } "#; - let result = flatten_str(src, Path::new("."), FlattenOpts::default()).unwrap(); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()).unwrap(); assert_eq!(result.matches(r#"\version"#).count(), 1); assert!(result.contains(r#"\version "2.24.0""#)); assert!(!result.contains(r#"\version "2.22.0""#)); } #[test] - fn test_duplicate_language_last_wins() { + fn test_every_language_stays() { + // Each applies from where it stands: dropping the first would read + // `c'` in German. let src = r#"\language "english" -{ c' } +{ cs' } \language "deutsch" -{ d' } +{ cis' } "#; - let result = flatten_str(src, Path::new("."), FlattenOpts::default()).unwrap(); - assert_eq!(result.matches(r#"\language"#).count(), 1); - assert!(result.contains(r#"\language "deutsch""#)); - assert!(!result.contains(r#"\language "english""#)); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()).unwrap(); + assert_eq!(result, src); } #[test] @@ -632,18 +665,16 @@ mod tests { } { c' } "#; - let result = flatten_str(src, Path::new("."), FlattenOpts::default()); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()); assert!(result.is_ok()); } #[test] - fn test_multiple_headers_error() { - let src = r#"\header { title = "A" } -\header { title = "B" } -{ c' } -"#; - let err = flatten_str(src, Path::new("."), FlattenOpts::default()).unwrap_err(); - assert!(matches!(err, FlattenError::MultipleHeaders)); + fn test_several_headers_stay() { + // LilyPond merges top-level headers; a score's own is its own. + let src = "\\header { title = \"A\" }\n\\header { composer = \"B\" }\n\\score { \\header { piece = \"I\" } { c' } }\n"; + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()).unwrap(); + assert_eq!(result, src); } // ----------------------------------------------------------------------- @@ -656,7 +687,7 @@ mod tests { { c' } "#; // No file foo.ly exists — should not error because the include is commented out. - let result = flatten_str(src, Path::new("."), FlattenOpts::default()); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()); assert!(result.is_ok()); let text = result.unwrap(); assert!(!text.contains("BEGIN INCLUDE")); @@ -665,9 +696,60 @@ mod tests { #[test] fn test_include_in_block_comment_skipped() { let src = "%{\n\\include \"foo.ly\"\n%}\n{ c' }\n"; - let result = flatten_str(src, Path::new("."), FlattenOpts::default()); + let result = flatten_str(src, Some(Path::new(".")), FlattenOpts::default()); assert!(result.is_ok()); let text = result.unwrap(); assert!(!text.contains("BEGIN INCLUDE")); } + + #[test] + fn test_includes_are_found_on_the_tree() { + let dir = TempDir::new().unwrap(); + write(dir.path(), "part.ily", "d'4\n"); + let opts = || FlattenOpts { + add_markers: false, + ..Default::default() + }; + // Mid-line, and after a one-line block comment (which left the line + // scanner "inside a comment" for the rest of the file). + let src = "%{ note %}\n{ c'4 \\include \"part.ily\" e'4 }\n"; + let result = flatten_str(src, Some(dir.path()), opts()).unwrap(); + assert_eq!(result, "%{ note %}\n{ c'4 \nd'4\n e'4 }\n"); + // Not in a string, and a commented one is not followed. + let src = "{ c'4^\"\\\\include \\\"x.ly\\\"\" } % \\include \"y.ly\"\n"; + assert_eq!(flatten_str(src, Some(dir.path()), opts()).unwrap(), src); + } + + #[test] + fn test_lilypond_files_and_missing_ones_are_kept() { + let src = "\\include \"english.ly\"\n{ cs'4 }\n"; + assert_eq!(flatten_str(src, None, FlattenOpts::default()).unwrap(), src); + let src = "\\include \"lib.ily\"\n{ c'4 }\n"; + assert!(matches!( + flatten_str(src, None, FlattenOpts::default()), + Err(FlattenError::NotFound(_)) + )); + let keep = FlattenOpts { + keep_missing: true, + ..Default::default() + }; + assert_eq!(flatten_str(src, None, keep).unwrap(), src); + } + + #[test] + fn test_the_map_tells_where_each_piece_comes_from() { + let dir = TempDir::new().unwrap(); + write(dir.path(), "part.ily", "d'4\n"); + let src = "{ c'4 }\n\\include \"part.ily\"\n{ e'4 }\n"; + let opts = FlattenOpts { + add_markers: false, + ..Default::default() + }; + let (out, map) = flatten_mapped(src, Some(dir.path()), &opts).unwrap(); + assert_eq!(out, "{ c'4 }\nd'4\n{ e'4 }\n"); + let starts: Vec = map.iter().map(|(at, _)| *at).collect(); + assert_eq!(starts, [0, 8, 12]); + assert!(matches!(map[1].1, Origin::Include { ref statement, .. } if statement == &(8..27))); + assert!(matches!(map[2].1, Origin::Source(28))); + } } diff --git a/src/adapters/ly_to_ir/builtins.rs b/src/adapters/ly_to_ir/builtins.rs index 6cc5b22..0cd288e 100644 --- a/src/adapters/ly_to_ir/builtins.rs +++ b/src/adapters/ly_to_ir/builtins.rs @@ -1,6 +1,7 @@ //! The commands LilyPond 2.26.0 defines, without their backslash: //! lexer keywords, the identifiers of its `ly/` init files and its markup -//! commands (1347). Generated: `python3 scripts/ly_builtins.py lilypond v2.26.0`. +//! commands (1347); and the files of its `ly/` directory (62). +//! Generated: `python3 scripts/ly_builtins.py lilypond v2.26.0`. /// Sorted, for binary search. pub(super) const BUILTINS: &[&str] = &[ @@ -1352,3 +1353,69 @@ pub(super) const BUILTINS: &[&str] = &[ "|", "~", ]; + +/// Files LilyPond itself provides to `\include`, sorted. +pub(crate) const LILYPOND_FILES: &[&str] = &[ + "ancient-init.ly", + "arabic.ly", + "articulate.ly", + "bagpipe.ly", + "base-tkit.ly", + "catalan.ly", + "chord-modifiers-init.ly", + "chord-repetition-init.ly", + "context-mods-init.ly", + "declarations-init.ly", + "deutsch.ly", + "drumpitch-init.ly", + "dynamic-scripts-init.ly", + "english.ly", + "engraver-init.ly", + "espanol.ly", + "event-listener.ly", + "generate-documentation.ly", + "grace-init.ly", + "graphviz-init.ly", + "gregorian.ly", + "guile-debugger.ly", + "hel-arabic.ly", + "init.ly", + "internal-backend-test.ly", + "italiano.ly", + "lilypond-book-preamble.ly", + "lyrics-tkit.ly", + "makam.ly", + "midi-init.ly", + "music-functions-init.ly", + "nederlands.ly", + "norsk.ly", + "paper-defaults-init.ly", + "performer-init.ly", + "persian.ly", + "piano-tkit.ly", + "portugues.ly", + "predefined-fretboards-init.ly", + "predefined-guitar-fretboards.ly", + "predefined-guitar-ninth-fretboards.ly", + "predefined-mandolin-fretboards.ly", + "predefined-ukulele-fretboards.ly", + "property-init.ly", + "satb.ly", + "scale-definitions-init.ly", + "scheme-sandbox.ly", + "script-init.ly", + "spanners-init.ly", + "ssaattbb.ly", + "staff-tkit.ly", + "string-tunings-init.ly", + "suomi.ly", + "svenska.ly", + "swing.ly", + "text-replacements.ly", + "titling-init.ly", + "toc-init.ly", + "turkish-makam.ly", + "vlaams.ly", + "vocal-tkit.ly", + "voice-tkit.ly", +]; diff --git a/src/adapters/ly_to_ir/mod.rs b/src/adapters/ly_to_ir/mod.rs index 390281f..eefd372 100644 --- a/src/adapters/ly_to_ir/mod.rs +++ b/src/adapters/ly_to_ir/mod.rs @@ -35,12 +35,14 @@ mod postprocess; mod state; mod syntax; mod text; +mod tokens; +mod version; mod walk; #[cfg(test)] mod tests; -use std::path::Path; +use std::path::{Path, PathBuf}; use num::rational::Ratio; @@ -54,6 +56,7 @@ use crate::ir::score::{Score, ScoreChild}; use crate::ir::Part; use crate::parser::LilyPondParser; +use super::ly_flatten::{flatten_mapped, FlattenError, FlattenOpts, Origin, SourceMap}; use super::{AdapterError, Result, ToIrAdapter}; use crate::ir::duration::{Duration, Frac}; @@ -226,13 +229,22 @@ pub fn read_source(path: &Path) -> Result { /// Check LilyPond text. By default only the syntax, from the tree (fast, no /// reading); with `semantic`, everything a reading reports too. Input the /// reader refuses as too large is an error diagnostic here, not an `Err`. +/// See [`LyToIrAdapter::check_str`] to follow includes. pub fn check(text: &str, semantic: bool) -> Vec { - let too_large = |message: String| Diagnostic::whole(Severity::Error, "too-large", message); + check_plain(text, semantic, false) +} + +fn too_large(message: String) -> Diagnostic { + Diagnostic::whole(Severity::Error, "too-large", message) +} + +/// [`check`], of text flattened with include paths when `follows_includes`. +fn check_plain(text: &str, semantic: bool, follows_includes: bool) -> Vec { if semantic { - return match LyToIrAdapter::new().read_str(text) { + return match LyToIrAdapter::new().read_plain(text, follows_includes) { Ok(reading) => reading.diagnostics, Err(e) => { - let mut out = check(text, false); + let mut out = check_plain(text, false, follows_includes); out.push(too_large(e.to_string())); tidy(&mut out); out @@ -246,7 +258,59 @@ pub fn check(text: &str, semantic: bool) -> Vec { } } +/// `d`, found in the flattened text, located in `source`: in the source's +/// own text at its place, in an included file at the `\include` that +/// brought it in, naming the file. +fn locate(d: Diagnostic, map: &SourceMap, source: &str) -> Diagnostic { + let i = map + .partition_point(|(at, _)| *at <= d.start) + .saturating_sub(1); + let Some((piece, origin)) = map.get(i) else { + return d; + }; + let piece_end = map.get(i + 1).map_or(usize::MAX, |(at, _)| *at); + let (start, end, message) = match origin { + Origin::Source(at) => { + let end = d.end.clamp(d.start, piece_end); + (at + d.start - piece, at + end - piece, d.message) + } + Origin::Include { statement, path } => ( + statement.start, + statement.end, + format!("in `{path}`: {}", d.message), + ), + }; + Diagnostic::between(source, start, end, d.severity, d.code, message) +} + +/// Flatten options for a reading with include paths: missing files are +/// kept, for the walk to warn about. +fn flatten_opts(include_paths: &[PathBuf]) -> FlattenOpts { + FlattenOpts { + include_paths: include_paths.to_vec(), + add_markers: false, + keep_missing: true, + } +} + +/// A flatten failure as a reading error: I/O stays I/O. +fn flatten_err(e: FlattenError) -> AdapterError { + match e { + FlattenError::Io { path, source } => AdapterError::Io(std::io::Error::new( + source.kind(), + format!("{}: {source}", path.display()), + )), + FlattenError::TooLarge(_) => AdapterError::Unsupported(e.to_string()), + e => AdapterError::Parse(e.to_string()), + } +} + +pub(crate) use builtins::LILYPOND_FILES; pub use text::HeaderField; +pub use tokens::{source_stats, strip_comments, tokenize, SourceStats, Token, TokenKind}; +pub use version::{ + lilypond_version, set_lilypond_version, strip_lilypond_version, LilyPondVersion, +}; /// Every `\header` field of LilyPond text, in source order, from the syntax /// tree alone (nothing is read): its key, its value as text (strings decoded, @@ -265,15 +329,45 @@ pub fn header_fields(text: &str) -> Vec { /// Parses LilyPond source text using tree-sitter and produces an IR `Score`. pub struct LyToIrAdapter { language: PitchLanguage, + /// Where to look for `\include`d files; `None`: includes are not followed. + include_paths: Option>, } impl LyToIrAdapter { pub fn new() -> Self { Self { language: PitchLanguage::Nederlands, + include_paths: None, } } + /// Follow `\include`s: the text is flattened first, relative includes + /// resolved against the file's directory (when reading a file) and then + /// `include_paths`. An include not found is an `ignored-include` + /// warning; diagnostics in an included file are reported at its + /// `\include`. Without this, includes are not followed. + pub fn with_include_paths(mut self, include_paths: Vec) -> Self { + self.include_paths = Some(include_paths); + self + } + + /// [`check`], following includes when the adapter does: an included + /// file that cannot be read is an `Err`. + pub fn check_str(&self, text: &str, semantic: bool) -> Result> { + let Some(paths) = &self.include_paths else { + return Ok(check(text, semantic)); + }; + let (flat, map) = match flatten_mapped(text, None, &flatten_opts(paths)) { + Ok(flattened) => flattened, + Err(e @ FlattenError::Io { .. }) => return Err(flatten_err(e)), + Err(e) => return Ok(vec![too_large(e.to_string())]), + }; + Ok(check_plain(&flat, semantic, true) + .into_iter() + .map(|d| locate(d, &map, text)) + .collect()) + } + /// Set the default pitch language (overridden by `\language` in the source). pub fn with_language(mut self, lang: PitchLanguage) -> Self { self.language = lang; @@ -281,28 +375,52 @@ impl LyToIrAdapter { } /// Read LilyPond text: every movement, and the diagnostics. + pub fn read_str(&self, text: &str) -> Result { + self.read_text(text, None) + } + + /// Read a LilyPond file (see [`read_source`]); its includes, when + /// followed, are relative to its directory. + pub fn read_file(&self, path: &Path) -> Result { + self.read_text(&read_source(path)?, path.parent()) + } + + /// Read LilyPond text whose relative includes, when followed, are + /// relative to `base_dir`. + pub fn read_text(&self, text: &str, base_dir: Option<&Path>) -> Result { + let Some(paths) = &self.include_paths else { + return self.read_plain(text, false); + }; + let (flat, map) = + flatten_mapped(text, base_dir, &flatten_opts(paths)).map_err(flatten_err)?; + let mut reading = self.read_plain(&flat, true)?; + for list in [&mut reading.diagnostics, &mut reading.later_movements] { + *list = std::mem::take(list) + .into_iter() + .map(|d| locate(d, &map, text)) + .collect(); + } + Ok(reading) + } + + /// Read text as it is (flattened when `follows_includes`). /// /// The walk recurses once per nested music block, so it runs on a thread /// with a stack of its own: how deep a score may nest must not depend on /// the caller's thread (Rust test threads and many Python threads get 2 MB /// or less). A panic on that thread resumes on the caller's. - pub fn read_str(&self, text: &str) -> Result { + fn read_plain(&self, text: &str, follows_includes: bool) -> Result { std::thread::scope(|scope| { std::thread::Builder::new() .name("lytk-ly-reader".to_string()) .stack_size(WALK_STACK_BYTES) - .spawn_scoped(scope, || self.read_here(text))? + .spawn_scoped(scope, || self.read_here(text, follows_includes))? .join() .unwrap_or_else(|payload| std::panic::resume_unwind(payload)) }) } - /// Read a LilyPond file (see [`read_source`]). - pub fn read_file(&self, path: &Path) -> Result { - self.read_str(&read_source(path)?) - } - - fn read_here(&self, source: &str) -> Result { + fn read_here(&self, source: &str, follows_includes: bool) -> Result { let mut parser = LilyPondParser::new().map_err(|e| AdapterError::Parse(e.to_string()))?; let tree = parser .parse(source) @@ -313,6 +431,7 @@ impl LyToIrAdapter { let mut state = state::WalkState::new(source); state.root = Some(root); state.language = self.language; + state.follows_includes = follows_includes; walk::walk_program(&mut state, root); let refused = |reason: String| { @@ -330,8 +449,10 @@ impl LyToIrAdapter { scores.push(score); } // The book's header, last assignment first, where a movement leaves a - // field unset. + // field unset; the file's version. + let version = version::version_of(source, root).map(|v| v.to_string()); for score in &mut scores { + score.metadata.lilypond_version = version.clone(); for (key, value) in state.book_header.iter().rev() { walk::set_header_field(&mut score.metadata, key, value.clone(), false); } diff --git a/src/adapters/ly_to_ir/state.rs b/src/adapters/ly_to_ir/state.rs index 9b691f8..515a04f 100644 --- a/src/adapters/ly_to_ir/state.rs +++ b/src/adapters/ly_to_ir/state.rs @@ -203,6 +203,9 @@ pub(super) struct WalkState<'src> { /// Above 0 while music already walked once is walked for another purpose /// (a chord-mode block read as notes): the first walk reported it. pub(super) quiet: u32, + /// The text read was flattened with include paths: an `\include` left + /// in it names a file not found. + pub(super) follows_includes: bool, /// Names assigned at the top level (`name = …`), whatever their value. pub(super) assigned: HashSet, /// The top-level music expression being read, if any: LilyPond makes a @@ -270,6 +273,7 @@ impl<'src> WalkState<'src> { diagnostics: Vec::new(), columns: Columns::default(), quiet: 0, + follows_includes: false, assigned: HashSet::new(), open_movement: None, fixed_octaves: 0, @@ -380,11 +384,17 @@ impl<'src> WalkState<'src> { let shown = file .map(|n| format!(" {}", self.text(n))) .unwrap_or_default(); - self.warn( - node, - "ignored-include", - format!("`\\include{shown}` is not followed: flatten the file first"), - ); + let why = if path + .as_deref() + .is_some_and(|p| super::builtins::LILYPOND_FILES.binary_search(&p).is_ok()) + { + "is one of LilyPond's own files, which lytk does not read" + } else if self.follows_includes { + "is not followed: no such file in the include paths" + } else { + "is not followed: flatten the file first, or give include paths" + }; + self.warn(node, "ignored-include", format!("`\\include{shown}` {why}")); } } diff --git a/src/adapters/ly_to_ir/syntax.rs b/src/adapters/ly_to_ir/syntax.rs index 3824894..1d3a9fa 100644 --- a/src/adapters/ly_to_ir/syntax.rs +++ b/src/adapters/ly_to_ir/syntax.rs @@ -6,6 +6,8 @@ use tree_sitter::{Node, Tree}; use crate::diagnostics::{Columns, Diagnostic, Severity}; use crate::ir::language::{parse_pitch_name, PitchLanguage}; +use super::version::{self, LilyPondVersion}; + /// Every syntax error in `tree`, parsed from `source`. pub(crate) fn syntax_diagnostics(tree: &Tree, source: &str) -> Vec { let mut out = Vec::new(); @@ -35,6 +37,8 @@ pub(crate) fn syntax_diagnostics(tree: &Tree, source: &str) -> Vec { } else if node.kind() == "embedded_scheme" { out.extend(after_scheme(cols, source, node)); out.extend(markup_override(cols, source, node)); + } else if version::is_statement(source, node) { + out.extend(invalid_version(cols, source, node)); } if descend && cursor.goto_first_child() { continue; @@ -251,6 +255,28 @@ fn markup_override(cols: &mut Columns, source: &str, node: Node) -> Option Option { + let statement = version::statement(word); + let (node, message) = match (statement.string, statement.value(source)) { + (Some(string), Some(value)) => match value.parse::() { + Ok(_) => return None, + Err(e) => (string, e), + }, + _ => (word, "quoted string expected after `\\version`".to_string()), + }; + Some(Diagnostic::counted( + cols, + source, + node, + node.end_byte(), + Severity::Error, + "invalid-version", + message, + )) +} + /// Whether `node` stands in markup, where any word or mark is text: it, or a /// block around it, follows `\markup` through markup commands and their /// Scheme arguments only (`\markup \hspace #1 .`, `\markup { \line { … } }`). diff --git a/src/adapters/ly_to_ir/tokens.rs b/src/adapters/ly_to_ir/tokens.rs new file mode 100644 index 0000000..27469d6 --- /dev/null +++ b/src/adapters/ly_to_ir/tokens.rs @@ -0,0 +1,327 @@ +//! Tokens of LilyPond text: the syntax tree's leaves, with strings, comments +//! and quoted identifiers kept whole. Embedded Scheme is tokenized as Scheme, +//! and LilyPond embedded in it (`#{ … #}`) as LilyPond. + +use tree_sitter::Node; + +use crate::diagnostics::Columns; +use crate::parser::LilyPondParser; + +use super::{AdapterError, Result}; + +/// What a [`Token`] is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TokenKind { + /// `% …` or `%{ … %}`; in Scheme `; …`, `#| … |#` or `#;datum`. + Comment, + /// `"…"`, whole. + String, + /// The `#`, `$`, `#@` or `$@` that starts an embedded Scheme expression, + /// whose tokens follow. + Scheme, + /// A backslashed word or mark: `\relative`, `\!`, `\(`, `\1`. + Command, + /// A word: a note name, a context, a lyric. + Symbol, + /// `4`, `2.5`. + Number, + /// `3/4`. + Fraction, + /// Brackets, octave marks, articulations and the other marks; Scheme's + /// brackets and quotes. + Punctuation, + /// Scheme's `#t`, `#f`, `#true`, `#false`. + Boolean, + /// A Scheme character: `#\a`, `#\space`. + Character, + /// A Scheme keyword: `#:key`. + Keyword, + /// Text the grammar cannot make a token of. + Error, +} + +impl TokenKind { + pub fn as_str(self) -> &'static str { + match self { + TokenKind::Comment => "comment", + TokenKind::String => "string", + TokenKind::Scheme => "scheme", + TokenKind::Command => "command", + TokenKind::Symbol => "symbol", + TokenKind::Number => "number", + TokenKind::Fraction => "fraction", + TokenKind::Punctuation => "punctuation", + TokenKind::Boolean => "boolean", + TokenKind::Character => "character", + TokenKind::Keyword => "keyword", + TokenKind::Error => "error", + } + } +} + +/// A token of LilyPond text: `text[start..end]` (bytes), at a 1-based line +/// and character column. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Token { + pub kind: TokenKind, + pub start: usize, + pub end: usize, + pub line: usize, + pub column: usize, + /// Whether the token is Scheme (embedded Scheme, its `#` or `$` + /// included) rather than LilyPond. + pub scheme: bool, +} + +/// The kind of token `node` is, if it is one (a node the walk does not +/// descend into); `None` for a node made of tokens. +fn kind_of(node: Node, text: &str) -> Option { + let kind = match node.kind() { + "comment" | "scheme_comment" => TokenKind::Comment, + "string" | "scheme_string" => TokenKind::String, + "embedded_scheme_prefix" => TokenKind::Scheme, + "quoted_identifier" + | "escaped_word" + | "dynamic" + | "ligature" + | "instrument_string_number" => TokenKind::Command, + "symbol" | "scheme_symbol" => TokenKind::Symbol, + "unsigned_integer" | "decimal_number" | "scheme_number" => TokenKind::Number, + "fraction" => TokenKind::Fraction, + "scheme_boolean" => TokenKind::Boolean, + "scheme_character" => TokenKind::Character, + "scheme_keyword" => TokenKind::Keyword, + _ if node.child_count() > 0 => return None, + _ if node.is_error() => TokenKind::Error, + _ if text.starts_with('\\') => TokenKind::Command, + _ => TokenKind::Punctuation, + }; + Some(kind) +} + +fn parse(text: &str) -> Result { + LilyPondParser::new() + .and_then(|mut p| p.parse(text)) + .map_err(|e| AdapterError::Parse(e.to_string())) +} + +/// Every token of `text`, in order. Whitespace is no token; every other +/// character is in exactly one. +pub fn tokenize(text: &str) -> Result> { + let tree = parse(text)?; + let mut out = Vec::new(); + let mut columns = Columns::default(); + let mut cursor = tree.walk(); + // Whether each ancestor of the current node is Scheme. + let mut parents: Vec = Vec::new(); + loop { + let node = cursor.node(); + let scheme = match node.kind() { + "embedded_scheme" => true, + "scheme_embedded_lilypond_text" => false, + _ => parents.last().copied().unwrap_or(false), + }; + let (start, end) = (node.start_byte(), node.end_byte()); + let kind = if node.is_missing() || start == end { + None + } else { + kind_of(node, text.get(start..end).unwrap_or("")) + }; + if let Some(kind) = kind { + let point = node.start_position(); + out.push(Token { + kind, + start, + end, + line: point.row + 1, + column: columns.column(text, start - point.column, start), + scheme, + }); + } else if cursor.goto_first_child() { + parents.push(scheme); + continue; + } + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() { + return Ok(out); + } + parents.pop(); + } + } +} + +/// `text` without its comments, LilyPond's and embedded Scheme's. A block +/// comment between two tokens becomes a space, as LilyPond reads it; a line +/// comment leaves its line break. +pub fn strip_comments(text: &str) -> Result { + let mut out = String::with_capacity(text.len()); + let mut at = 0; + for token in tokenize(text)? { + if token.kind != TokenKind::Comment { + continue; + } + out.push_str(&text[at..token.start]); + let glued = |c: Option| c.is_some_and(|c| !c.is_whitespace()); + if glued(text[..token.start].chars().next_back()) && glued(text[token.end..].chars().next()) + { + out.push(' '); + } + at = token.end; + } + out.push_str(&text[at..]); + Ok(out) +} + +/// Counts of LilyPond source text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceStats { + pub bytes: usize, + pub lines: usize, + pub tokens: usize, + pub comments: usize, + /// Embedded Scheme expressions. + pub scheme: usize, + /// Text the grammar cannot tokenize. + pub error_tokens: usize, +} + +/// The counts of `text`, from its tokens. +pub fn source_stats(text: &str) -> Result { + let tokens = tokenize(text)?; + let count = |kind| tokens.iter().filter(|t| t.kind == kind).count(); + Ok(SourceStats { + bytes: text.len(), + lines: text.lines().count(), + tokens: tokens.len(), + comments: count(TokenKind::Comment), + scheme: count(TokenKind::Scheme), + error_tokens: count(TokenKind::Error), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kinds(text: &str) -> Vec<(&'static str, &str)> { + tokenize(text) + .unwrap() + .into_iter() + .map(|t| (t.kind.as_str(), &text[t.start..t.end])) + .collect() + } + + #[test] + fn scheme_is_tokenized_as_scheme() { + let src = "#(define x \"a\") ##f $y #(list #\\a #:k 'q ; c\n #{ c'4 #})"; + let tokens: Vec<(&str, &str, bool)> = tokenize(src) + .unwrap() + .into_iter() + .map(|t| (t.kind.as_str(), &src[t.start..t.end], t.scheme)) + .collect(); + assert_eq!( + tokens, + [ + ("scheme", "#", true), + ("punctuation", "(", true), + ("symbol", "define", true), + ("symbol", "x", true), + ("string", "\"a\"", true), + ("punctuation", ")", true), + ("scheme", "#", true), + ("boolean", "#f", true), + ("scheme", "$", true), + ("symbol", "y", true), + ("scheme", "#", true), + ("punctuation", "(", true), + ("symbol", "list", true), + ("character", "#\\a", true), + ("keyword", "#:k", true), + ("punctuation", "'", true), + ("symbol", "q", true), + ("comment", "; c", true), + ("punctuation", "#{", true), + ("symbol", "c", false), + ("punctuation", "'", false), + ("number", "4", false), + ("punctuation", "#}", true), + ("punctuation", ")", true), + ] + ); + } + + #[test] + fn leaves_become_tokens() { + assert_eq!( + kinds("\\relative c'' { \\time 3/4 c4.\\p( d) % end\n}"), + [ + ("command", "\\relative"), + ("symbol", "c"), + ("punctuation", "'"), + ("punctuation", "'"), + ("punctuation", "{"), + ("command", "\\time"), + ("fraction", "3/4"), + ("symbol", "c"), + ("number", "4"), + ("punctuation", "."), + ("command", "\\p"), + ("punctuation", "("), + ("symbol", "d"), + ("punctuation", ")"), + ("comment", "% end"), + ("punctuation", "}"), + ] + ); + assert_eq!( + kinds("t = \"a \\\"b\" #(define x 1) \\< \\!"), + [ + ("symbol", "t"), + ("punctuation", "="), + ("string", "\"a \\\"b\""), + ("scheme", "#"), + ("punctuation", "("), + ("symbol", "define"), + ("symbol", "x"), + ("number", "1"), + ("punctuation", ")"), + ("command", "\\<"), + ("command", "\\!"), + ] + ); + } + + #[test] + fn source_stats_count_tokens_by_kind() { + let s = source_stats("% a\n#(define x 1)\n{ c'4 %{b%} }\n").unwrap(); + assert_eq!( + ( + s.bytes, + s.lines, + s.tokens, + s.comments, + s.scheme, + s.error_tokens + ), + (32, 3, 13, 2, 1, 0) + ); + } + + #[test] + fn positions_count_characters() { + let t = tokenize("é %{x%}\n d").unwrap(); + assert_eq!( + t.iter().map(|t| (t.line, t.column)).collect::>(), + [(1, 1), (1, 3), (2, 3)] + ); + } + + #[test] + fn comments_are_stripped_as_lilypond_skips_them() { + let src = "c4%{x%}d4 % tail\n#(display \"%\" ; s\n#|b|#1)\n"; + assert_eq!( + strip_comments(src).unwrap(), + "c4 d4 \n#(display \"%\" \n1)\n" + ); + } +} diff --git a/src/adapters/ly_to_ir/version.rs b/src/adapters/ly_to_ir/version.rs new file mode 100644 index 0000000..202db87 --- /dev/null +++ b/src/adapters/ly_to_ir/version.rs @@ -0,0 +1,284 @@ +//! `\version`: the LilyPond version a file states, read and edited on the +//! syntax tree, so a commented-out statement does not count. + +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::str::FromStr; + +use tree_sitter::Node; + +use crate::parser::LilyPondParser; + +use super::{AdapterError, Result}; + +/// A LilyPond version, compared numerically (`2.24` is `2.24.0`). Accepted as +/// LilyPond 2.24 accepts it (`parse-lily-version` in `scm/lily-library.scm`): +/// `major.minor.patch`, with an optional fourth part (any text, which is not +/// compared), or `major.minor` for a stable series (an even minor). The +/// numbers are ASCII digits. +#[derive(Debug, Clone)] +pub struct LilyPondVersion { + pub major: u32, + pub minor: u32, + pub patch: u32, + pub extra: Option, +} + +impl LilyPondVersion { + fn key(&self) -> (u32, u32, u32) { + (self.major, self.minor, self.patch) + } +} + +impl FromStr for LilyPondVersion { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + let invalid = || format!("invalid version string \"{s}\""); + let number = |p: &str| { + p.bytes() + .all(|b| b.is_ascii_digit()) + .then(|| p.parse::().ok()) + .flatten() + .ok_or_else(invalid) + }; + match s.split('.').collect::>()[..] { + [major, minor] => { + let (major, minor) = (number(major)?, number(minor)?); + if minor % 2 == 1 { + return Err(format!( + "version \"{s}\" leaves out the third number, which only a stable \ + release (an even second number) may" + )); + } + Ok(Self { + major, + minor, + patch: 0, + extra: None, + }) + } + [major, minor, patch, ref extra @ ..] if extra.len() <= 1 => Ok(Self { + major: number(major)?, + minor: number(minor)?, + patch: number(patch)?, + extra: extra.first().map(|e| e.to_string()), + }), + _ => Err(invalid()), + } + } +} + +impl fmt::Display for LilyPondVersion { + /// `2.24.0`, the fourth part after a dot when there is one. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch)?; + match &self.extra { + Some(extra) => write!(f, ".{extra}"), + None => Ok(()), + } + } +} + +impl PartialEq for LilyPondVersion { + fn eq(&self, other: &Self) -> bool { + self.key() == other.key() + } +} + +impl Eq for LilyPondVersion {} + +impl Hash for LilyPondVersion { + fn hash(&self, state: &mut H) { + self.key().hash(state); + } +} + +impl PartialOrd for LilyPondVersion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for LilyPondVersion { + fn cmp(&self, other: &Self) -> Ordering { + self.key().cmp(&other.key()) + } +} + +/// A `\version` statement: the command, and the string after it when one +/// follows directly (LilyPond allows only whitespace between them). +pub(crate) struct Statement<'t> { + pub word: Node<'t>, + pub string: Option>, +} + +impl Statement<'_> { + /// The text between the quotes, as LilyPond's lexer takes it (no escapes). + pub fn value<'s>(&self, source: &'s str) -> Option<&'s str> { + let raw = source.get(self.string?.byte_range())?; + raw.strip_prefix('"') + .map(|r| r.strip_suffix('"').unwrap_or(r)) + } + + fn end(&self) -> usize { + self.string.unwrap_or(self.word).end_byte() + } +} + +/// Whether `node` is a `\version` command. +pub(crate) fn is_statement(source: &str, node: Node) -> bool { + node.kind() == "escaped_word" && source.get(node.byte_range()) == Some("\\version") +} + +/// The `\version` statement `word` starts. +pub(crate) fn statement<'t>(word: Node<'t>) -> Statement<'t> { + let string = word.next_sibling().filter(|n| n.kind() == "string"); + Statement { word, string } +} + +/// Every `\version` statement under `root`, in source order. +fn statements<'t>(source: &str, root: Node<'t>) -> Vec> { + let mut out = Vec::new(); + let mut cursor = root.walk(); + loop { + let node = cursor.node(); + if is_statement(source, node) { + out.push(statement(node)); + } + if cursor.goto_first_child() { + continue; + } + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() { + return out; + } + } + } +} + +/// The version the first `\version` statement under `root` states, if valid. +pub(crate) fn version_of(source: &str, root: Node) -> Option { + statements(source, root) + .first()? + .value(source)? + .parse() + .ok() +} + +fn parse(text: &str) -> Result { + LilyPondParser::new() + .and_then(|mut p| p.parse(text)) + .map_err(|e| AdapterError::Parse(e.to_string())) +} + +/// The version the first `\version` statement of `text` states: `None` when +/// there is none, or when it is not a valid version (`lytk check` reports +/// it). +pub fn lilypond_version(text: &str) -> Option { + let tree = parse(text).ok()?; + version_of(text, tree.root_node()) +} + +/// `text` with every `\version` statement stating `version`, written as +/// given, or with one added at the top (after a byte-order mark) when there is +/// none. `version` must be a valid [`LilyPondVersion`]. +pub fn set_lilypond_version(text: &str, version: &str) -> Result { + version + .parse::() + .map_err(AdapterError::Parse)?; + let statement = format!("\\version \"{version}\""); + let tree = parse(text)?; + let found = statements(text, tree.root_node()); + if found.is_empty() { + let (bom, rest) = match text.strip_prefix('\u{feff}') { + Some(rest) => ("\u{feff}", rest), + None => ("", text), + }; + return Ok(format!("{bom}{statement}\n{rest}")); + } + let mut out = text.to_string(); + for s in found.iter().rev() { + out.replace_range(s.word.start_byte()..s.end(), &statement); + } + Ok(out) +} + +/// `text` without its `\version` statements. A statement alone on its line +/// takes the line with it. +pub fn strip_lilypond_version(text: &str) -> Result { + let tree = parse(text)?; + let mut out = text.to_string(); + for s in statements(text, tree.root_node()).iter().rev() { + let (mut start, mut end) = (s.word.start_byte(), s.end()); + let line_start = text[..start].rfind('\n').map_or(0, |i| i + 1); + let rest = &text[end..]; + let trailing = rest.len() - rest.trim_start_matches([' ', '\t']).len(); + let after = &rest[trailing..]; + let newline = [("\r\n", 2), ("\n", 1)] + .into_iter() + .find(|(nl, _)| after.starts_with(nl)) + .map(|(_, n)| n); + if text[line_start..start].trim().is_empty() && (newline.is_some() || after.is_empty()) { + start = line_start; + end += trailing + newline.unwrap_or(0); + } + out.replace_range(start..end, ""); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(s: &str) -> LilyPondVersion { + s.parse().unwrap() + } + + #[test] + fn versions_parse_and_compare_as_lilypond_does() { + assert_eq!(v("2.24"), v("2.24.0")); + assert!(v("2.24.0") < v("2.24.1") && v("2.24.10") > v("2.24.9")); + assert_eq!(v("2.25.3.foo").to_string(), "2.25.3.foo"); + assert_eq!(v("2.25.3.foo"), v("2.25.3")); + for bad in [ + "2.25", + "2", + "2.x.0", + "", + "2..0", + "2.24.0.a.b", + " 2.24.0", + "-2.24.0", + ] { + assert!(bad.parse::().is_err(), "{bad:?}"); + } + } + + #[test] + fn statements_are_found_on_the_tree_and_edited() { + let src = "% \\version \"1.0.0\"\n\\version \"2.24.0\"\n{ c'4 }\n"; + assert_eq!(lilypond_version(src), Some(v("2.24.0"))); + assert_eq!(lilypond_version("{ c'4 }"), None); + assert_eq!(lilypond_version("\\version \"2.x\" { c'4 }"), None); + assert_eq!( + set_lilypond_version(src, "2.26").unwrap(), + "% \\version \"1.0.0\"\n\\version \"2.26\"\n{ c'4 }\n" + ); + assert_eq!( + set_lilypond_version("\u{feff}{ c'4 }", "2.24.0").unwrap(), + "\u{feff}\\version \"2.24.0\"\n{ c'4 }" + ); + assert!(set_lilypond_version(src, "2.25").is_err()); + assert_eq!( + strip_lilypond_version(src).unwrap(), + "% \\version \"1.0.0\"\n{ c'4 }\n" + ); + assert_eq!( + strip_lilypond_version("{ c'4 } \\version \"2.24.0\" % x").unwrap(), + "{ c'4 } % x" + ); + } +} diff --git a/src/adapters/ly_to_ir/walk.rs b/src/adapters/ly_to_ir/walk.rs index dfacc55..68c610c 100644 --- a/src/adapters/ly_to_ir/walk.rs +++ b/src/adapters/ly_to_ir/walk.rs @@ -40,8 +40,11 @@ fn walk_top<'a>(state: &mut WalkState<'a>, parent: Node<'a>) { let text = state.text(node); match text { "\\version" => { - // Skip version string; \version "2.24.0" consumes next string - i += 1; // skip string + // Its string: the syntax check judges it, and + // read_here reads it from the tree. + if children.get(i + 1).is_some_and(|n| n.kind() == "string") { + i += 1; + } } "\\language" => { // \language "english" diff --git a/src/diagnostics.rs b/src/diagnostics.rs index 87b3baf..dc6eec3 100644 --- a/src/diagnostics.rs +++ b/src/diagnostics.rs @@ -92,6 +92,28 @@ impl Diagnostic { } } + /// A diagnostic from byte `start` to byte `end` of `source`. + pub fn between( + source: &str, + start: usize, + end: usize, + severity: Severity, + code: &'static str, + message: String, + ) -> Self { + let before = source.get(..start).unwrap_or(""); + let line_start = before.rfind('\n').map_or(0, |i| i + 1); + Self { + severity, + code, + message, + line: before.matches('\n').count() + 1, + column: before[line_start..].chars().count() + 1, + start, + end, + } + } + /// A diagnostic about the input as a whole, at its start. pub fn whole(severity: Severity, code: &'static str, message: String) -> Self { Self { @@ -123,7 +145,7 @@ pub(crate) struct Columns { impl Columns { /// The 1-based character column of byte `start` on the line starting at /// byte `line_start`. - fn column(&mut self, source: &str, line_start: usize, start: usize) -> usize { + pub(crate) fn column(&mut self, source: &str, line_start: usize, start: usize) -> usize { if line_start != self.line_start || start < self.byte { (self.line_start, self.byte, self.chars) = (line_start, line_start, 0); } diff --git a/src/ir/mod.rs b/src/ir/mod.rs index 21465f8..0aabb64 100644 --- a/src/ir/mod.rs +++ b/src/ir/mod.rs @@ -41,6 +41,7 @@ pub mod note; pub mod part; pub mod pitch; pub mod score; +pub mod stats; pub mod voice; // Layer 1: Music tree (format-agnostic representation). diff --git a/src/ir/score.rs b/src/ir/score.rs index 28f8a7f..40b3a8e 100644 --- a/src/ir/score.rs +++ b/src/ir/score.rs @@ -32,6 +32,10 @@ pub struct ScoreMetadata { pub pitch_mode: PitchMode, /// Anacrusis / pickup duration (emitted as `\partial `). pub partial_duration: Option, + /// The LilyPond version the source's `\version` states (`2.24.0`), when + /// the score was read from LilyPond that states a valid one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lilypond_version: Option, } /// Page layout dimensions (all measurements in cm, staff-size in points). diff --git a/src/ir/stats.rs b/src/ir/stats.rs new file mode 100644 index 0000000..fe67600 --- /dev/null +++ b/src/ir/stats.rs @@ -0,0 +1,104 @@ +//! Statistics of a score: what `lytk.info` and `lytk info --json` report. + +use std::collections::BTreeSet; + +use super::duration::Frac; +use super::note::{Note, VoiceElement}; +use super::score::Score; +use super::Part; + +/// A part's counts. +#[derive(Debug, Clone, PartialEq)] +pub struct PartInfo { + pub measures: usize, + /// Distinct voice numbers in its measures. + pub voices: usize, + /// Sounding notes: chord members and grace notes each count. + pub notes: usize, +} + +/// A score's counts. +#[derive(Debug, Clone, PartialEq)] +pub struct ScoreInfo { + pub parts: Vec, + pub note_count: usize, + pub voice_count: usize, + /// Bars of the score (its longest part's). + pub bar_count: usize, + /// Length in quarter notes: each bar as long as its longest voice. + pub duration_quarters: Frac, + /// Lyric syllables. + pub lyric_count: usize, + /// Chord symbols (harmonies). + pub chord_symbol_count: usize, + pub grace_note_count: usize, +} + +fn notes(element: &VoiceElement) -> &[Note] { + match element { + VoiceElement::Note(n) => std::slice::from_ref(n.as_ref()), + VoiceElement::Chord(c) => &c.notes, + VoiceElement::Rest(_) => &[], + } +} + +fn part_length(part: &Part) -> Frac { + part.measures + .iter() + .map(|m| { + m.voices + .iter() + .map(|v| { + v.elements + .iter() + .map(VoiceElement::metric_duration) + .sum::() + }) + .max() + .unwrap_or_default() + }) + .sum() +} + +/// The counts of `score`. +pub fn score_info(score: &Score) -> ScoreInfo { + let parts = score.parts(); + let mut info = ScoreInfo { + parts: Vec::new(), + note_count: 0, + voice_count: 0, + bar_count: parts.iter().map(|p| p.measures.len()).max().unwrap_or(0), + duration_quarters: parts + .iter() + .map(|p| part_length(p)) + .max() + .unwrap_or_default() + * Frac::from_integer(4), + lyric_count: 0, + chord_symbol_count: 0, + grace_note_count: 0, + }; + for part in parts { + let mut voices = BTreeSet::new(); + let mut part_notes = 0; + for measure in &part.measures { + info.chord_symbol_count += measure.harmonies.len(); + for voice in &measure.voices { + voices.insert(voice.number); + for note in voice.elements.iter().flat_map(notes) { + part_notes += 1; + info.lyric_count += note.lyrics.len(); + info.grace_note_count += usize::from(note.is_grace); + } + } + } + info.note_count += part_notes; + info.voice_count += voices.len(); + info.parts.push(PartInfo { + measures: part.measures.len(), + voices: voices.len(), + notes: part_notes, + }); + } + info +} diff --git a/src/lytk/__init__.py b/src/lytk/__init__.py index 09fdd26..bb74f68 100644 --- a/src/lytk/__init__.py +++ b/src/lytk/__init__.py @@ -8,6 +8,7 @@ HeaderField, InternalError, LilyPondSyntaxError, + LilyPondVersion, LytkError, Measure, MusicDocument, @@ -17,20 +18,25 @@ Pitch, Rest, Score, + Token, Voice, change_language, check_lilypond, compute_metrics, flatten, + flatten_string, from_abc, from_abc_string, from_abc_tunes, + from_abc_tunes_string, from_humdrum, from_humdrum_string, from_event_sequence, from_lilypond, from_lilypond_movements, + from_lilypond_movements_string, from_lilypond_music, + from_lilypond_music_movements, from_lilypond_music_string, from_lilypond_string, from_musicxml, @@ -39,8 +45,14 @@ from_note_array, from_piano_roll, header_fields, + info, invert, + lilypond_version, retrograde, + set_lilypond_version, + source_stats, + strip_comments, + strip_lilypond_version, to_abc, to_humdrum, to_event_sequence, @@ -50,6 +62,7 @@ to_mxl_bytes, to_note_array, to_piano_roll, + tokenize, transpose, transpose_interval, transpose_to_key, @@ -66,6 +79,19 @@ "check_lilypond", "HeaderField", "header_fields", + # LilyPond \version + "LilyPondVersion", + "lilypond_version", + "set_lilypond_version", + "strip_lilypond_version", + # LilyPond tokens + "Token", + "tokenize", + "strip_comments", + # Statistics + "info", + "source_stats", + # Scores "MusicDocument", "Score", # Structured note navigation (read-only Layer-2 tree) @@ -83,12 +109,15 @@ "from_lilypond", "from_lilypond_string", "from_lilypond_movements", + "from_lilypond_movements_string", "from_lilypond_music", + "from_lilypond_music_movements", "from_lilypond_music_string", "from_abc", "from_humdrum", "from_abc_string", "from_abc_tunes", + "from_abc_tunes_string", "from_humdrum_string", "to_lilypond", "to_lilypond_music", @@ -98,6 +127,7 @@ "to_humdrum", # LilyPond \include flattening "flatten", + "flatten_string", # Transforms "transpose", "transpose_interval", diff --git a/src/lytk/_core.pyi b/src/lytk/_core.pyi index d586fa9..bd333c9 100644 --- a/src/lytk/_core.pyi +++ b/src/lytk/_core.pyi @@ -47,7 +47,7 @@ class Diagnostic: def code(self) -> str: """A stable identifier: ``syntax-error``, ``missing-token``, ``invalid-duration``, ``invalid-ratio``, ``not-lilypond``, - ``too-large`` (errors); ``unknown-command``, ``unrecognized-token``, + ``too-large``, ``invalid-version`` (errors); ``unknown-command``, ``unrecognized-token``, ``ignored-include``, ``unknown-language``, ``dropped-music``, ``skipped-score``, ``unsupported-value`` (warnings).""" ... @@ -85,6 +85,11 @@ class Score: @property def lyricist(self) -> str | None: ... @property + def lilypond_version(self) -> LilyPondVersion | None: + """The version the source's ``\\version`` states, for a score read + from LilyPond that states a valid one.""" + ... + @property def language(self) -> str | None: ... @property def num_parts(self) -> int: ... @@ -138,6 +143,10 @@ class MusicDocument: @property def lyricist(self) -> str | None: ... @property + def lilypond_version(self) -> LilyPondVersion | None: + """As :attr:`Score.lilypond_version`.""" + ... + @property def header(self) -> dict[str, str]: """Every header field, as :attr:`Score.header`.""" ... @@ -347,11 +356,107 @@ def header_fields(text: str) -> list[HeaderField]: out.""" ... -def check_lilypond(text: str, *, semantic: bool = False) -> list[Diagnostic]: +class LilyPondVersion: + """A LilyPond version, compared numerically (``2.24`` equals ``2.24.0``). + Accepted as LilyPond 2.24 accepts it: ``major.minor.patch`` with an + optional fourth part (kept, not compared), or ``major.minor`` with an even + minor. Other strings raise :class:`ParseError`. Hashable, picklable.""" + + def __init__(self, text: str) -> None: ... + @property + def major(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def patch(self) -> int: ... + @property + def extra(self) -> str | None: + """The fourth part, if any.""" + ... + def __lt__(self, other: LilyPondVersion) -> bool: ... + def __le__(self, other: LilyPondVersion) -> bool: ... + def __gt__(self, other: LilyPondVersion) -> bool: ... + def __ge__(self, other: LilyPondVersion) -> bool: ... + def __hash__(self) -> int: ... + +def lilypond_version(text: str) -> LilyPondVersion | None: + """The version the first ``\\version`` statement states, from the parse + tree (a commented-out one does not count); *None* when there is none or + it is invalid (``check_lilypond`` reports ``invalid-version``).""" + ... + +def set_lilypond_version(text: str, version: LilyPondVersion | str) -> str: + """*text* with every ``\\version`` statement stating *version* (a string + is written as given, once checked), or with one added at the top.""" + ... + +def strip_lilypond_version(text: str) -> str: + """*text* without its ``\\version`` statements.""" + ... + +class Token: + """A token of LilyPond text (see :func:`tokenize`). Hashable.""" + + @property + def kind(self) -> str: + """``comment``, ``string``, ``command``, ``symbol``, ``number``, + ``fraction``, ``punctuation``, ``scheme`` (the ``#`` or ``$`` that + starts embedded Scheme), ``boolean``, ``character``, ``keyword`` + (Scheme's) or ``error``.""" + ... + @property + def scheme(self) -> bool: + """Whether the token is Scheme rather than LilyPond.""" + ... + @property + def text(self) -> str: ... + @property + def start(self) -> int: + """Character offset: ``text[t.start:t.end]`` is the token.""" + ... + @property + def end(self) -> int: ... + @property + def line(self) -> int: + """1-based.""" + ... + @property + def column(self) -> int: + """1-based, in characters.""" + ... + +def tokenize(text: str) -> list[Token]: + """The tokens of LilyPond text, from the parse tree; strings and comments + whole, embedded Scheme as Scheme tokens. Every character but whitespace + is in exactly one token, ``error`` tokens holding what the grammar cannot + tokenize.""" + ... + +def strip_comments(text: str) -> str: + """*text* without its comments, LilyPond's and embedded Scheme's.""" + ... + +def source_stats(text: str) -> dict[str, int]: + """Counts of LilyPond source text from its tokens: ``bytes``, ``lines``, + ``tokens``, ``comments``, ``scheme``, ``error_tokens``.""" + ... + +def info(score: Score) -> dict[str, Any]: + """A score's metadata and counts, as ``lytk info --json`` prints them: + ``title`` … ``language``, ``lilypond_version``; ``part_count``, + ``note_count``, ``voice_count``, ``bar_count``, ``duration_quarters``, + ``lyric_count``, ``chord_symbol_count``, ``grace_note_count``; and + ``parts``, a dict per part.""" + ... + +def check_lilypond( + text: str, *, semantic: bool = False, include_paths: list[str] | None = None +) -> list[Diagnostic]: """Diagnostics of LilyPond text, in source order: the syntax only (fast, nothing is read), or with ``semantic=True`` what a reading reports too. Never raises for bad input: input too large to read is a ``too-large`` - error.""" + error. *include_paths* follows includes as the readers do; an included + file that cannot be read raises ``OSError``.""" ... def from_musicxml(path: str) -> Score: ... @@ -360,25 +465,67 @@ def from_musicxml_bytes(data: bytes) -> Score: """Parse MusicXML or compressed MXL from in-memory bytes.""" ... def from_lilypond( - path: str, *, language: str | None = None, strict: bool = False + path: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, ) -> Score: """The first movement; ``strict=True`` raises LilyPondSyntaxError on an - error. The diagnostics are in ``Score.diagnostics`` either way.""" + error. The diagnostics are in ``Score.diagnostics`` either way. With + *include_paths*, ``\\include`` statements are followed (the file's directory, + then the paths); a diagnostic in an included file is reported at its + ``\\include``, and one not found is an ``ignored-include`` warning.""" ... def from_lilypond_string( - text: str, *, language: str | None = None, strict: bool = False + text: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, ) -> Score: ... def from_lilypond_movements( - path: str, *, language: str | None = None, strict: bool = False + path: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, ) -> list[Score]: """Every movement of a LilyPond file: one score per ``\\score`` block and per top-level music expression, each with the file's diagnostics.""" ... +def from_lilypond_movements_string( + text: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, +) -> list[Score]: + """Every movement of LilyPond text, each with the text's diagnostics.""" + ... +def from_lilypond_music_movements( + path: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, +) -> list[MusicDocument]: + """Every movement of a LilyPond file as a Music tree, each with the + file's diagnostics.""" + ... def from_lilypond_music( - path: str, *, language: str | None = None, strict: bool = False + path: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, ) -> MusicDocument: ... def from_lilypond_music_string( - text: str, *, language: str | None = None, strict: bool = False + text: str, + *, + language: str | None = None, + strict: bool = False, + include_paths: list[str] | None = None, ) -> MusicDocument: ... def to_lilypond( score: Score, @@ -386,17 +533,33 @@ def to_lilypond( *, language: str | None = None, relative: bool | None = None, + version: LilyPondVersion | str | None = None, ) -> str: """*relative*: ``True`` for ``\\relative`` entry, ``False`` for absolute, ``None`` for the score's own.""" ... def to_lilypond_music( - doc: MusicDocument, path: str | None = None + doc: MusicDocument, + path: str | None = None, + *, + version: LilyPondVersion | str | None = None, ) -> str: ... def to_musicxml(score: Score, path: str | None = None) -> str: ... def to_mxl_bytes(score: Score) -> bytes: """Serialize a score to compressed MusicXML (a ZIP archive).""" ... +def flatten_string( + text: str, + *, + base_dir: str | None = None, + include_paths: list[str] | None = None, + add_markers: bool = True, +) -> str: + """Expand the ``\\include`` directives of LilyPond text (relative ones + against *base_dir*, then *include_paths*), found on the parse tree. + ParseError for a missing file (LilyPond's own, like ``english.ly``, stay + as they are), a circular include or an expansion past the bounds.""" + ... def flatten( input: str, output: str | None = None, @@ -408,6 +571,9 @@ def from_abc(path: str) -> Score: ... def from_abc_string(text: str) -> Score: ... def from_abc_tunes(path: str) -> list[Score]: """Every tune of an ABC file, one score each (``from_abc`` reads the first).""" +def from_abc_tunes_string(text: str) -> list[Score]: + """Every tune of an ABC string, one score each.""" + ... def to_abc(score: Score, path: str | None = None) -> str: ... def from_humdrum(path: str) -> Score: ... def from_humdrum_string(text: str) -> Score: ... diff --git a/src/lytk/cli.py b/src/lytk/cli.py index f99a4f7..cd11485 100644 --- a/src/lytk/cli.py +++ b/src/lytk/cli.py @@ -480,47 +480,23 @@ def info( input: Annotated[str, typer.Argument(help="Input file.", show_default=False)], as_json: Annotated[bool, typer.Option("--json", help="Machine-readable JSON.")] = False, ) -> None: - """Show a score's metadata and parts.""" - score = read_score(input) - parts = score.iter_parts() + """Show a score's metadata, counts and parts (``lytk.info``).""" + data = lytk.info(read_score(input)) if as_json: - _json( - { - "title": score.title, - "subtitle": score.subtitle, - "composer": score.composer, - "arranger": score.arranger, - "lyricist": score.lyricist, - "language": score.language, - "part_count": len(parts), - "note_count": len(_pitches(score)), - "parts": [ - { - "id": p.part_id, - "name": p.name, - "abbreviation": p.abbreviation, - "measures": len(p.measures), - "staves": p.staves, - "midi_program": p.midi_program, - "midi_instrument": p.midi_instrument, - } - for p in parts - ], - } - ) + _json(data) return - for label, value in [ - ("Title", score.title), - ("Composer", score.composer), - ("Subtitle", score.subtitle), - ("Arranger", score.arranger), - ("Language", score.language), + for label, key in [ + ("Title", "title"), + ("Composer", "composer"), + ("Subtitle", "subtitle"), + ("Arranger", "arranger"), + ("Language", "language"), ]: - if value: - typer.echo(f"{label + ':':<10}{value}") - typer.echo(f"{'Parts:':<10}{len(parts)}") - for p in parts: - typer.echo(f" - {p.name or p.part_id} ({len(p.measures)} measures)") + if data[key]: + typer.echo(f"{label + ':':<10}{data[key]}") + typer.echo(f"{'Parts:':<10}{data['part_count']}") + for p in data["parts"]: + typer.echo(f" - {p['name'] or p['id']} ({p['measures']} measures)") def _length(elements: list[Any]) -> Fraction: diff --git a/src/lytk/datasets/__init__.py b/src/lytk/datasets/__init__.py index 4167f83..1791f46 100644 --- a/src/lytk/datasets/__init__.py +++ b/src/lytk/datasets/__init__.py @@ -1,18 +1,21 @@ """Dataset utilities for symbolic-music ML pipelines (Epic F). Provides a lazy :class:`Dataset` base, a :class:`FolderDataset` that loads a -directory of music files, representation converters (note-array / event / -piano-roll), train/val/test splits, on-disk caching, and torch/TensorFlow -adapters — datasets and padded data loaders — imported lazily so neither -framework is needed unless used. +directory of music files, a :class:`RecordsDataset` over records holding +music as text (JSON Lines), representation converters (note-array / event / +piano-roll), train/val/test splits (by ratio, by group, or the records' own), +on-disk caching, and torch/TensorFlow adapters — datasets and padded data +loaders — imported lazily so neither framework is needed unless used. """ from __future__ import annotations from lytk.datasets.base import ( SUPPORTED_EXTENSIONS, + TEXT_FORMATS, Dataset, FolderDataset, + RecordsDataset, Subset, load_document, pad_collate, @@ -20,8 +23,10 @@ __all__ = [ "SUPPORTED_EXTENSIONS", + "TEXT_FORMATS", "Dataset", "FolderDataset", + "RecordsDataset", "Subset", "load_document", "pad_collate", diff --git a/src/lytk/datasets/base.py b/src/lytk/datasets/base.py index 2f95a71..2cc7ee2 100644 --- a/src/lytk/datasets/base.py +++ b/src/lytk/datasets/base.py @@ -1,57 +1,150 @@ -"""Dataset base classes, the folder dataset, splits, caching and ML adapters.""" +"""Dataset base classes, the folder and records datasets, splits, caching and +ML adapters.""" from __future__ import annotations import functools import hashlib +import json +import os import random +import tempfile +import warnings from pathlib import Path -from typing import Any, Callable, Iterator, Sequence +from typing import Any, Callable, Hashable, Iterable, Iterator, Sequence import numpy as np from lytk import _core -# Recognised input extensions → loader producing a MusicDocument. Keep in step -# with `_EXT_FORMAT` in `lytk.cli`; `test_datasets.py` asserts they agree. +# Recognised input extensions → format. Keep in step with `_EXT_FORMAT` in +# `lytk.cli`; `test_datasets.py` asserts they agree. _LY_EXTS = {".ly", ".ily"} _XML_EXTS = {".xml", ".musicxml", ".mxl"} _MIDI_EXTS = {".mid", ".midi"} _ABC_EXTS = {".abc"} _KERN_EXTS = {".krn", ".kern"} SUPPORTED_EXTENSIONS = _LY_EXTS | _XML_EXTS | _MIDI_EXTS | _ABC_EXTS | _KERN_EXTS - - -def load_document(path: str | Path): - """Load any supported music file as a :class:`MusicDocument`. +_FORMAT_OF_EXT = { + **dict.fromkeys(_LY_EXTS, "lilypond"), + **dict.fromkeys(_XML_EXTS, "musicxml"), + **dict.fromkeys(_MIDI_EXTS, "midi"), + **dict.fromkeys(_ABC_EXTS, "abc"), + **dict.fromkeys(_KERN_EXTS, "humdrum"), +} +# Formats a record's text can hold (MIDI is binary). +TEXT_FORMATS = ("lilypond", "musicxml", "abc", "humdrum") + +_ON_ERROR = ("raise", "skip", "warn") +_MOVEMENTS = ("first", "all") +# What reading an item can raise for a bad item: lytk's errors (ParseError is +# also a ValueError), I/O, and a value no reader accepts. +_READ_ERRORS = (_core.LytkError, OSError, ValueError) + + +def _check_choice(name: str, value: str, choices: Sequence[str]) -> str: + if value not in choices: + raise ValueError(f"{name} must be one of {list(choices)}, not {value!r}") + return value + + +def _read( + source: str | Path, + fmt: str, + *, + from_text: bool, + movements: str, + language: str | None, + include_paths: Sequence[str] | None, + strict: bool, + quantize: int | None, +) -> list: + """The documents of a file (``from_text=False``) or of text: its first + movement, or all of them. The LilyPond options apply to LilyPond only, + ``quantize`` to MIDI only.""" + every = movements == "all" + src = str(source) + if fmt == "lilypond": + opts: dict[str, Any] = {"language": language, "strict": strict} + if include_paths is not None: + opts["include_paths"] = [str(p) for p in include_paths] + if from_text: + if every: + scores = _core.from_lilypond_movements_string(src, **opts) + return [s.to_music_document() for s in scores] + return [_core.from_lilypond_music_string(src, **opts)] + if every: + return _core.from_lilypond_music_movements(src, **opts) + return [_core.from_lilypond_music(src, **opts)] + if fmt == "abc": + if every: + tunes = _core.from_abc_tunes_string(src) if from_text else _core.from_abc_tunes(src) + return [t.to_music_document() for t in tunes] + score = _core.from_abc_string(src) if from_text else _core.from_abc(src) + elif fmt == "musicxml": + score = _core.from_musicxml_string(src) if from_text else _core.from_musicxml(src) + elif fmt == "humdrum": + score = _core.from_humdrum_string(src) if from_text else _core.from_humdrum(src) + elif fmt == "midi" and not from_text: + score = _core.from_midi(src, quantize=quantize) + else: + raise ValueError(f"unsupported format: {fmt!r}") + return [score.to_music_document()] + + +def load_document( + path: str | Path, + *, + language: str | None = None, + include_paths: Sequence[str] | None = None, + strict: bool = False, + quantize: int | None = None, +): + """Load any supported music file as a :class:`MusicDocument` (its first + movement). LilyPond is parsed directly into the Layer-1 Music tree; every other format - is parsed into a Score and lifted. + is parsed into a Score and lifted. ``language``, ``include_paths`` and + ``strict`` are passed to the LilyPond reader, ``quantize`` to the MIDI one. """ path = Path(path) ext = path.suffix.lower() - if ext in _LY_EXTS: - return _core.from_lilypond_music(str(path)) - if ext in _XML_EXTS: - return _core.from_musicxml(str(path)).to_music_document() - if ext in _MIDI_EXTS: - return _core.from_midi(str(path)).to_music_document() - if ext in _ABC_EXTS: - return _core.from_abc(str(path)).to_music_document() - if ext in _KERN_EXTS: - return _core.from_humdrum(str(path)).to_music_document() - raise ValueError( - f"unsupported file extension: {ext!r} ({path}); " - f"supported: {sorted(SUPPORTED_EXTENSIONS)}" + if ext not in _FORMAT_OF_EXT: + raise ValueError( + f"unsupported file extension: {ext!r} ({path}); " + f"supported: {sorted(SUPPORTED_EXTENSIONS)}" + ) + (doc,) = _read( + path, + _FORMAT_OF_EXT[ext], + from_text=False, + movements="first", + language=language, + include_paths=include_paths, + strict=strict, + quantize=quantize, ) + return doc -# Representation converters: name → (function, default kwargs). +# Representation converters, and their defaults: the cache key normalizes +# arguments with them, so `to_note_arrays()` and +# `iter_representation("note_array")` share cache entries. _REPRESENTATIONS: dict[str, Callable[..., np.ndarray]] = { "note_array": _core.to_note_array, "event_sequence": _core.to_event_sequence, "piano_roll": _core.to_piano_roll, } +_DEFAULTS: dict[str, dict[str, Any]] = { + "note_array": {"resolution": 480}, + "event_sequence": { + "resolution": 480, + "max_time_shift": 100, + "velocity_bins": 32, + "encode_velocity": True, + }, + "piano_roll": {"resolution": 480, "encode_velocity": True}, +} def _require_representation(representation: str) -> Callable[..., np.ndarray]: @@ -74,11 +167,12 @@ def pad_collate(batch: Sequence[Any], pad_value: float = 0.0): ``RuntimeError: stack expects each tensor to be equal size``. This pads the first axis to the longest item in the batch. - Returns ``(padded, lengths)``. The lengths are returned rather than left to - be inferred from ``pad_value``, because the padding value is not reserved: - ``0`` is a legitimate event code, pitch and velocity, so trailing zeros are - genuinely ambiguous. Use them to build a mask or to - ``pack_padded_sequence``. + Returns ``(padded, lengths)``, or ``(padded, lengths, ids)`` when the items + are ``(tensor, id)`` pairs (a dataset made with ``return_ids=True``). The + lengths are returned rather than left to be inferred from ``pad_value``, + because the padding value is not reserved: ``0`` is a legitimate event + code, pitch and velocity, so trailing zeros are genuinely ambiguous. Use + them to build a mask or to ``pack_padded_sequence``. Suitable as a ``collate_fn`` for any ``torch.utils.data.DataLoader``: @@ -93,9 +187,13 @@ def pad_collate(batch: Sequence[Any], pad_value: float = 0.0): except ImportError as exc: # pragma: no cover - optional dep raise ImportError("PyTorch is required for pad_collate()") from exc - tensors = [torch.as_tensor(np.asarray(item)) for item in batch] + with_ids = bool(batch) and isinstance(batch[0], tuple) + items = [item[0] for item in batch] if with_ids else batch + tensors = [torch.as_tensor(np.asarray(item)) for item in items] lengths = torch.tensor([t.shape[0] for t in tensors], dtype=torch.long) padded = pad_sequence(tensors, batch_first=True, padding_value=pad_value) + if with_ids: + return padded, lengths, [item[1] for item in batch] return padded, lengths @@ -105,17 +203,53 @@ class Dataset: Subclasses implement :meth:`__len__` and :meth:`__getitem__`. Everything else (representation conversion, metrics, splits, torch/tf adapters) is built on top of those two. + + ``on_error`` says what iterating does with an item that cannot be read: + ``"raise"`` (the default) raises, ``"skip"`` leaves it out and records it + in :attr:`errors`, ``"warn"`` does the same and warns. Iteration covers + ``for doc in ds``, :meth:`iter_representation`, :meth:`to_representation`, + :meth:`metrics` and the TensorFlow adapters; indexing (``ds[i]``, the + torch adapters) always raises for such an item. """ + on_error: str = "raise" + def __len__(self) -> int: # pragma: no cover - abstract raise NotImplementedError def __getitem__(self, index: int): # pragma: no cover - abstract raise NotImplementedError + @property + def ids(self) -> list[str]: + """An identifier per item, in order (here the index as a string).""" + return [str(i) for i in range(len(self))] + + @property + def errors(self) -> dict[str, str]: + """The items iteration skipped: id → ``"ErrorType: message"``.""" + return self.__dict__.setdefault("_errors", {}) + + def _failed(self, index: int, error: Exception) -> None: + """Handle an item that cannot be read, as ``on_error`` says.""" + if self.on_error == "raise": + raise error + item_id = self.ids[index] + message = f"{type(error).__name__}: {error}" + self.errors[item_id] = message + if self.on_error == "warn": + warnings.warn(f"lytk: skipped {item_id}: {message}", RuntimeWarning, stacklevel=4) + + def _each(self, index: int, fn: Callable[[int], Any]) -> Iterator[Any]: + """``fn(index)``, or nothing when the item cannot be read.""" + try: + yield fn(index) + except _READ_ERRORS as error: + self._failed(index, error) + def __iter__(self) -> Iterator: for i in range(len(self)): - yield self[i] + yield from self._each(i, self.__getitem__) # -- representation conversion ------------------------------------------- @@ -130,20 +264,66 @@ def iter_representation( memory, so it scales to large folders. When a ``cache_dir`` is set the per-item cache is consulted as each item is produced. """ + for _, arr in self._iter_converted(representation, kwargs): + yield arr + + def _iter_converted( + self, representation: str, kwargs: dict[str, Any] + ) -> Iterator[tuple[int, np.ndarray]]: + """(index, array) for every item that can be read.""" _require_representation(representation) # validate the name up front for i in range(len(self)): - yield self._convert_item(i, representation, kwargs) + for arr in self._each(i, lambda j: self._convert_item(j, representation, kwargs)): + yield i, arr def to_representation(self, representation: str, **kwargs: Any) -> list[np.ndarray]: - """Eagerly convert every item to ``representation`` (one array per item). + """Eagerly convert every item to ``representation`` (one array per item + that can be read). This materialises the whole dataset; use :meth:`iter_representation` for a memory-bounded stream over large corpora. """ return list(self.iter_representation(representation, **kwargs)) + def _cache_dir(self) -> Path | None: + """Where converted items are cached; ``None`` for no cache.""" + return None + + def _cache_source(self, index: int) -> bytes: # pragma: no cover - abstract + """What item ``index`` is read from, and how, for the cache key.""" + raise NotImplementedError + def _convert_item(self, index, representation, kwargs) -> np.ndarray: - return _REPRESENTATIONS[representation](self[index], **kwargs) + convert = _REPRESENTATIONS[representation] + cache_dir = self._cache_dir() + if cache_dir is None: + return convert(self[index], **kwargs) + # The key: the item's source (content and reading options), lytk's + # version and the converter's arguments with their defaults filled in. + params = json.dumps( + { + "lytk": _core.__version__, + "representation": representation, + "kwargs": {**_DEFAULTS.get(representation, {}), **kwargs}, + }, + sort_keys=True, + ) + digest = hashlib.sha256(params.encode("utf-8") + b"\0" + self._cache_source(index)) + cache_file = cache_dir / f"{digest.hexdigest()}.npy" + if cache_file.exists(): + return np.load(cache_file, allow_pickle=False) + arr = np.asarray(convert(self[index], **kwargs)) + # Written to a temporary file and renamed: a reader never sees half a + # file, and two writers of one entry write the same bytes. + fd, tmp = tempfile.mkstemp(dir=cache_dir, suffix=".npy.tmp") + try: + with os.fdopen(fd, "wb") as out: + np.save(out, arr, allow_pickle=False) + os.replace(tmp, cache_file) + except BaseException: + Path(tmp).unlink(missing_ok=True) + raise + return arr def to_note_arrays(self, resolution: int = 480) -> list[np.ndarray]: return self.to_representation("note_array", resolution=resolution) @@ -159,44 +339,77 @@ def to_pianorolls(self, resolution: int = 480, encode_velocity: bool = True) -> def metrics( self, resolution: int = 480, measure_resolution: int | None = None ) -> list[dict[str, Any]]: - """Compute objective metrics for every item.""" + """Compute objective metrics for every item that can be read.""" return [ - _core.compute_metrics(self[i], resolution, measure_resolution) - for i in range(len(self)) + _core.compute_metrics(doc, resolution, measure_resolution) + for doc in self ] # -- splitting ----------------------------------------------------------- + def _group_keys(self, groups: Any) -> list[Hashable]: + """The group of each item: ``groups`` is one key per item, or a + function of the item's id.""" + if callable(groups): + return [groups(item_id) for item_id in self.ids] + if isinstance(groups, str): + raise TypeError("groups is a field name only for a RecordsDataset") + keys = list(groups) + if len(keys) != len(self): + raise ValueError(f"groups has {len(keys)} keys for {len(self)} items") + return keys + def split( self, ratios: Sequence[float] = (0.8, 0.1, 0.1), seed: int = 0, + *, + groups: Any = None, ) -> tuple["Subset", ...]: """Deterministically partition into subsets by the given ratios. - Returns one :class:`Subset` per ratio (e.g. train/val/test). + Returns one :class:`Subset` per ratio (e.g. train/val/test). With + ``groups`` (one key per item, or a function of the item's id), items of + a group land in the same subset: the groups are shuffled and each goes + to the subset furthest below its share of the items. """ if not ratios or any(r < 0 for r in ratios): raise ValueError("ratios must be non-empty and non-negative") n = len(self) - indices = list(range(n)) - random.Random(seed).shuffle(indices) total = float(sum(ratios)) - subsets: list[Subset] = [] - start = 0 - for r in ratios[:-1]: - end = start + int(round(n * r / total)) - subsets.append(Subset(self, indices[start:end])) - start = end - # The last subset takes the remainder so every item is used. - subsets.append(Subset(self, indices[start:])) - return tuple(subsets) + if groups is None: + indices = list(range(n)) + random.Random(seed).shuffle(indices) + subsets: list[Subset] = [] + start = 0 + for r in ratios[:-1]: + end = start + int(round(n * r / total)) + subsets.append(Subset(self, indices[start:end])) + start = end + # The last subset takes the remainder so every item is used. + subsets.append(Subset(self, indices[start:])) + return tuple(subsets) + members: dict[Hashable, list[int]] = {} + for i, key in enumerate(self._group_keys(groups)): + members.setdefault(key, []).append(i) + order = list(members) + random.Random(seed).shuffle(order) + targets = [n * r / total for r in ratios] + parts: list[list[int]] = [[] for _ in ratios] + for key in order: + # The subset furthest below its share (the first on a tie). + best = max(range(len(parts)), key=lambda k: (targets[k] - len(parts[k]), -k)) + parts[best].extend(members[key]) + return tuple(Subset(self, part) for part in parts) # -- ML framework adapters (lazy imports) -------------------------------- - def to_pytorch_dataset(self, representation: str = "note_array", **kwargs: Any): + def to_pytorch_dataset( + self, representation: str = "note_array", *, return_ids: bool = False, **kwargs: Any + ): """Return a ``torch.utils.data.Dataset`` yielding tensors of the - chosen representation. Requires PyTorch. + chosen representation, or ``(tensor, id)`` pairs with ``return_ids``. + Requires PyTorch. Conversion is lazy: each item is converted (and cached, if a ``cache_dir`` is set) only when it is indexed, so constructing the @@ -210,6 +423,7 @@ def to_pytorch_dataset(self, representation: str = "note_array", **kwargs: Any): _require_representation(representation) # validate the name up front outer = self + ids = self.ids if return_ids else None class _LytkTorchDataset(TorchDataset): def __len__(self) -> int: @@ -217,17 +431,21 @@ def __len__(self) -> int: def __getitem__(self, i: int): arr = outer._convert_item(i, representation, kwargs) - return torch.as_tensor(np.asarray(arr)) + tensor = torch.as_tensor(np.asarray(arr)) + return (tensor, ids[i]) if ids is not None else tensor return _LytkTorchDataset() - def to_tensorflow_dataset(self, representation: str = "note_array", **kwargs: Any): - """Return a ``tf.data.Dataset`` of the chosen representation. - Requires TensorFlow. + def to_tensorflow_dataset( + self, representation: str = "note_array", *, return_ids: bool = False, **kwargs: Any + ): + """Return a ``tf.data.Dataset`` of the chosen representation, or of + ``(tensor, id)`` pairs with ``return_ids``. Requires TensorFlow. The dataset streams items through a generator, converting (and caching, if a ``cache_dir`` is set) one item at a time rather than - building the whole corpus up front. + building the whole corpus up front. Items that cannot be read are + skipped or raise as ``on_error`` says. """ try: import tensorflow as tf @@ -236,19 +454,22 @@ def to_tensorflow_dataset(self, representation: str = "note_array", **kwargs: An _require_representation(representation) # validate the name up front outer = self - n = len(outer) + ids = self.ids if return_ids else None def _gen(): - for i in range(n): - yield np.asarray(outer._convert_item(i, representation, kwargs)) - - if n: - # Convert a single probe item to derive the tensor spec; this is - # one item, not the whole dataset (and it warms the cache). - probe = np.asarray(outer._convert_item(0, representation, kwargs)) - spec = tf.TensorSpec(shape=[None] * probe.ndim, dtype=probe.dtype) + for i, arr in outer._iter_converted(representation, kwargs): + yield (np.asarray(arr), ids[i]) if ids is not None else np.asarray(arr) + + # One probe item derives the tensor spec: one item, not the whole + # dataset (and it warms the cache). + probe = next(self._iter_converted(representation, kwargs), None) + if probe is not None: + arr = np.asarray(probe[1]) + spec = tf.TensorSpec(shape=[None] * arr.ndim, dtype=arr.dtype) else: # pragma: no cover - empty dataset spec = tf.TensorSpec(shape=[None], dtype=tf.int32) + if ids is not None: + spec = (spec, tf.TensorSpec(shape=(), dtype=tf.string)) return tf.data.Dataset.from_generator(_gen, output_signature=spec) # -- ML framework data loaders (batching) -------------------------------- @@ -260,6 +481,7 @@ def to_pytorch_dataloader( batch_size: int = 1, shuffle: bool = False, pad_value: float = 0.0, + return_ids: bool = False, representation_kwargs: dict[str, Any] | None = None, **loader_kwargs: Any, ): @@ -267,7 +489,8 @@ def to_pytorch_dataloader( Like :meth:`to_pytorch_dataset` but batched, with :func:`pad_collate` wired in so that ``batch_size > 1`` works on ragged scores. Each batch - is a ``(padded, lengths)`` tuple. + is a ``(padded, lengths)`` tuple, ``(padded, lengths, ids)`` with + ``return_ids``. Unlike the other methods on this class, ``**kwargs`` here go to the ``DataLoader`` (``num_workers``, ``pin_memory``, ``drop_last``, …); @@ -292,7 +515,7 @@ def to_pytorch_dataloader( ) from exc dataset = self.to_pytorch_dataset( - representation, **(representation_kwargs or {}) + representation, return_ids=return_ids, **(representation_kwargs or {}) ) loader_kwargs.setdefault( "collate_fn", functools.partial(pad_collate, pad_value=pad_value) @@ -308,9 +531,11 @@ def to_tensorflow_dataloader( batch_size: int = 1, shuffle: bool = False, pad_value: float = 0.0, + return_ids: bool = False, representation_kwargs: dict[str, Any] | None = None, ): - """Return a batched ``tf.data.Dataset`` of ``(padded, lengths)`` tuples. + """Return a batched ``tf.data.Dataset`` of ``(padded, lengths)`` tuples, + ``(padded, lengths, ids)`` with ``return_ids``. The TensorFlow counterpart of :meth:`to_pytorch_dataloader`: pads each batch along the ragged first axis with ``padded_batch`` and carries the @@ -331,9 +556,14 @@ def to_tensorflow_dataloader( ) from exc base = self.to_tensorflow_dataset( - representation, **(representation_kwargs or {}) + representation, return_ids=return_ids, **(representation_kwargs or {}) ) - with_lengths = base.map(lambda item: (item, tf.shape(item)[0])) + if return_ids: + dtype = base.element_spec[0].dtype + with_lengths = base.map(lambda item, item_id: (item, tf.shape(item)[0], item_id)) + else: + dtype = base.element_spec.dtype + with_lengths = base.map(lambda item: (item, tf.shape(item)[0])) if shuffle: # Bounded by the dataset size: items are converted lazily, so the # buffer holds arrays, not documents. @@ -341,19 +571,19 @@ def to_tensorflow_dataloader( # Cast through numpy: every representation is an integer dtype # (int64 events, int32 note arrays, uint8 rolls) and TensorFlow refuses # to build an int constant from the float default, unlike torch. - dtype = base.element_spec.dtype pad_scalar = np.asarray(pad_value).astype(dtype.as_numpy_dtype) - return with_lengths.padded_batch( - batch_size, - padding_values=( - tf.constant(pad_scalar, dtype=dtype), - tf.constant(0, dtype=tf.int32), - ), + padding: tuple = ( + tf.constant(pad_scalar, dtype=dtype), + tf.constant(0, dtype=tf.int32), ) + if return_ids: + padding += (tf.constant("", dtype=tf.string),) + return with_lengths.padded_batch(batch_size, padding_values=padding) class Subset(Dataset): - """A view of a parent dataset restricted to a list of indices.""" + """A view of a parent dataset restricted to a list of indices. Its items, + ids, cache and errors are the parent's.""" def __init__(self, dataset: Dataset, indices: Sequence[int]): self.dataset = dataset @@ -365,13 +595,148 @@ def __len__(self) -> int: def __getitem__(self, index: int): return self.dataset[self.indices[index]] + @property + def ids(self) -> list[str]: + parent = self.dataset.ids + return [parent[i] for i in self.indices] + + @property + def on_error(self) -> str: # type: ignore[override] + return self.dataset.on_error + + @property + def errors(self) -> dict[str, str]: + return self.dataset.errors + + def _failed(self, index: int, error: Exception) -> None: + self.dataset._failed(self.indices[index], error) + + def _convert_item(self, index, representation, kwargs) -> np.ndarray: + return self.dataset._convert_item(self.indices[index], representation, kwargs) + + +class _SourceDataset(Dataset): + """What the folder and records datasets share: reading options, the + movements they make items of, and the cache.""" + + def _setup( + self, + *, + cache_dir: str | Path | None, + on_error: str, + movements: str, + language: str | None, + include_paths: Sequence[str | Path] | None, + strict: bool, + quantize: int | None, + ) -> None: + self.on_error = _check_choice("on_error", on_error, _ON_ERROR) + self.movements = _check_choice("movements", movements, _MOVEMENTS) + self._read_opts = { + "language": language, + "include_paths": None if include_paths is None else [str(p) for p in include_paths], + "strict": strict, + "quantize": quantize, + } + self.cache_dir = Path(cache_dir) if cache_dir is not None else None + if self.cache_dir is not None: + self.cache_dir.mkdir(parents=True, exist_ok=True) + # With movements="all": (source index, movement index) per item. + self._items: list[tuple[int, int]] | None = None + self._last: tuple[int, list] | None = None + + # A source is a file or a record; subclasses say how many there are, what + # identifies one, and how to read it. + def _n_sources(self) -> int: # pragma: no cover - abstract + raise NotImplementedError + + def _source_id(self, source: int) -> str: # pragma: no cover - abstract + raise NotImplementedError + + def _source_bytes(self, source: int) -> bytes: # pragma: no cover - abstract + raise NotImplementedError + + def _read_source(self, source: int, movements: str) -> list: # pragma: no cover + raise NotImplementedError + + def _documents(self, source: int) -> list: + """Every movement of a source (the last source read is kept).""" + if self._last is None or self._last[0] != source: + self._last = (source, self._read_source(source, "all")) + return self._last[1] + + def _item_list(self) -> list[tuple[int, int]]: + """With movements="all", every (source, movement), reading each source + once; a source that cannot be read makes no item (or raises, as + ``on_error`` says).""" + if self._items is None: + items: list[tuple[int, int]] = [] + for s in range(self._n_sources()): + try: + count = len(self._documents(s)) + except _READ_ERRORS as error: + if self.on_error == "raise": + raise + message = f"{type(error).__name__}: {error}" + self.errors[self._source_id(s)] = message + if self.on_error == "warn": + warnings.warn( + f"lytk: skipped {self._source_id(s)}: {message}", + RuntimeWarning, + stacklevel=3, + ) + continue + items.extend((s, k) for k in range(count)) + self._items = items + return self._items -class FolderDataset(Dataset): + def __len__(self) -> int: + if self.movements == "first": + return self._n_sources() + return len(self._item_list()) + + def __getitem__(self, index: int): + if self.movements == "first": + (doc,) = self._read_source(range(self._n_sources())[index], "first") + return doc + source, k = self._item_list()[index] + return self._documents(source)[k] + + @property + def ids(self) -> list[str]: + """The source's id per item; with ``movements="all"``, followed by + ``#`` and the movement's number, from 1.""" + if self.movements == "first": + return [self._source_id(s) for s in range(self._n_sources())] + return [f"{self._source_id(s)}#{k + 1}" for s, k in self._item_list()] + + def _cache_dir(self) -> Path | None: + return self.cache_dir + + def _cache_source(self, index: int) -> bytes: + if self.movements == "first": + source, movement = range(self._n_sources())[index], 0 + else: + source, movement = self._item_list()[index] + options = json.dumps( + {**self._read_opts, "movements": self.movements, "movement": movement}, + sort_keys=True, + ) + return options.encode("utf-8") + b"\0" + self._source_bytes(source) + + +class FolderDataset(_SourceDataset): """A dataset over the supported music files found in a directory. - Documents are loaded lazily on access. When ``cache_dir`` is given, - converted representations are cached to ``.npy`` files keyed by source - file and conversion parameters. + Documents are loaded lazily on access; an item's id is its path relative to + ``root``. ``movements="all"`` makes each movement of a file (each + ``\\score`` of a LilyPond file, each tune of an ABC file) an item. + ``language``, ``include_paths`` and ``strict`` are passed to the LilyPond + reader, ``quantize`` to the MIDI one. ``on_error`` is described in + :class:`Dataset`. When ``cache_dir`` is given, converted representations + are cached to ``.npy`` files keyed by the file's content, how it is read, + the lytk version and the conversion parameters (the content of included + files is not part of the key). """ def __init__( @@ -380,6 +745,13 @@ def __init__( recursive: bool = True, extensions: Sequence[str] | None = None, cache_dir: str | Path | None = None, + *, + on_error: str = "raise", + movements: str = "first", + language: str | None = None, + include_paths: Sequence[str | Path] | None = None, + strict: bool = False, + quantize: int | None = None, ): self.root = Path(root) if not self.root.is_dir(): @@ -389,46 +761,149 @@ def __init__( self.paths: list[Path] = sorted( p for p in globber("*") if p.is_file() and p.suffix.lower() in exts ) - self.cache_dir = Path(cache_dir) if cache_dir is not None else None - if self.cache_dir is not None: - self.cache_dir.mkdir(parents=True, exist_ok=True) - - def __len__(self) -> int: - return len(self.paths) - - def __getitem__(self, index: int): - return load_document(self.paths[index]) + self._rel = [p.relative_to(self.root).as_posix() for p in self.paths] + self._setup( + cache_dir=cache_dir, + on_error=on_error, + movements=movements, + language=language, + include_paths=include_paths, + strict=strict, + quantize=quantize, + ) @property def filenames(self) -> list[str]: return [p.name for p in self.paths] - def _path_hash(self, index: int) -> str: - """Stable short hash identifying the source file. + def _n_sources(self) -> int: + return len(self.paths) - Derived from the path relative to ``root`` (falling back to the - absolute path), so two files that share a stem but live in different - subfolders — or differ only by extension, e.g. ``train/foo.ly`` and - ``valid/foo.xml`` — get distinct cache keys instead of colliding. - """ - path = self.paths[index] - try: - rel = path.relative_to(self.root) - except ValueError: # pragma: no cover - path outside root - rel = path - return hashlib.sha1(rel.as_posix().encode("utf-8")).hexdigest()[:16] + def _source_id(self, source: int) -> str: + return self._rel[source] - def _convert_item(self, index, representation, kwargs) -> np.ndarray: - # On-disk cache of converted representations (EFT2). - convert = _REPRESENTATIONS[representation] - if self.cache_dir is None: - return convert(self[index], **kwargs) - key = "_".join(f"{k}-{v}" for k, v in sorted(kwargs.items())) - stem = self.paths[index].stem - path_hash = self._path_hash(index) - cache_file = self.cache_dir / f"{stem}__{representation}__{key}__{path_hash}.npy" - if cache_file.exists(): - return np.load(cache_file, allow_pickle=False) - arr = convert(self[index], **kwargs) - np.save(cache_file, np.asarray(arr), allow_pickle=False) - return arr + def _source_bytes(self, source: int) -> bytes: + return self.paths[source].read_bytes() + + def _read_source(self, source: int, movements: str) -> list: + path = self.paths[source] + fmt = _FORMAT_OF_EXT.get(path.suffix.lower()) + if fmt is None: + raise ValueError(f"unsupported file extension: {path.suffix!r} ({path})") + return _read(path, fmt, from_text=False, movements=movements, **self._read_opts) + + +class RecordsDataset(_SourceDataset): + """A dataset over records (dicts) holding music as text: each record's + ``text_field`` in ``format`` (``"lilypond"``, ``"musicxml"``, ``"abc"`` or + ``"humdrum"``), identified by its ``id_field``. + + Items keep their record (:meth:`record`), and their id is the record's + (followed by ``#`` and the movement's number with ``movements="all"``). + With ``split_field``, the records carry their own splits: :meth:`split` + returns them instead of re-shuffling by ratio. The other options are as in + :class:`FolderDataset`; the cache is keyed by the record's text. + """ + + def __init__( + self, + records: Iterable[dict[str, Any]], + *, + text_field: str = "text", + id_field: str = "id", + format: str = "lilypond", + split_field: str | None = None, + cache_dir: str | Path | None = None, + on_error: str = "raise", + movements: str = "first", + language: str | None = None, + include_paths: Sequence[str | Path] | None = None, + strict: bool = False, + ): + self.format = _check_choice("format", format, TEXT_FORMATS) + self.text_field, self.id_field, self.split_field = text_field, id_field, split_field + self.records: list[dict[str, Any]] = list(records) + seen: set[str] = set() + for n, record in enumerate(self.records): + for field in (text_field, id_field): + if field not in record: + raise ValueError(f"record {n} has no {field!r} field") + if not isinstance(record[text_field], str): + raise ValueError(f"record {n}: {text_field!r} is not a string") + record_id = str(record[id_field]) + if record_id in seen: + raise ValueError(f"duplicate record id {record_id!r}") + seen.add(record_id) + self._setup( + cache_dir=cache_dir, + on_error=on_error, + movements=movements, + language=language, + include_paths=include_paths, + strict=strict, + quantize=None, + ) + + @classmethod + def from_records(cls, records: Iterable[dict[str, Any]], **kwargs: Any) -> "RecordsDataset": + """A dataset over an iterable of records (keywords as the class).""" + return cls(records, **kwargs) + + @classmethod + def from_jsonl(cls, path: str | Path, **kwargs: Any) -> "RecordsDataset": + """A dataset over a JSON Lines file, one record per line (keywords as + the class).""" + with open(path, encoding="utf-8") as lines: + return cls((json.loads(line) for line in lines if line.strip()), **kwargs) + + def record(self, index: int) -> dict[str, Any]: + """The record item ``index`` comes from.""" + if self.movements == "first": + return self.records[index] + return self.records[self._item_list()[index][0]] + + def _n_sources(self) -> int: + return len(self.records) + + def _source_id(self, source: int) -> str: + return str(self.records[source][self.id_field]) + + def _source_bytes(self, source: int) -> bytes: + return f"{self.format}\0{self.records[source][self.text_field]}".encode("utf-8") + + def _read_source(self, source: int, movements: str) -> list: + text = self.records[source][self.text_field] + return _read(text, self.format, from_text=True, movements=movements, **self._read_opts) + + def _group_keys(self, groups: Any) -> list[Hashable]: + if isinstance(groups, str): + return [self.record(i).get(groups) for i in range(len(self))] + return super()._group_keys(groups) + + def split( # type: ignore[override] + self, + ratios: Sequence[float] | None = None, + seed: int = 0, + *, + groups: Any = None, + field: str | None = None, + ): + """The records' own splits, or a ratio split. + + With ``field`` (or ``split_field`` given to the dataset, when no + ``ratios`` are), returns ``{value: Subset}`` for each value of that + field, in order of first appearance: a curated, decontaminated split + stays as it is. Otherwise splits by ``ratios`` (default + ``(0.8, 0.1, 0.1)``) as :meth:`Dataset.split` does; ``groups`` may name + a field whose equal values stay together. + """ + if field is None and ratios is None: + field = self.split_field + if field is not None: + if ratios is not None: + raise ValueError("pass ratios or field, not both") + parts: dict[Any, list[int]] = {} + for i in range(len(self)): + parts.setdefault(self.record(i).get(field), []).append(i) + return {value: Subset(self, indices) for value, indices in parts.items()} + return super().split(ratios if ratios is not None else (0.8, 0.1, 0.1), seed, groups=groups) diff --git a/src/python.rs b/src/python.rs index e66aea4..99532f2 100644 --- a/src/python.rs +++ b/src/python.rs @@ -3,7 +3,7 @@ //! Every function and method here that reads, writes or transforms music runs //! inside [`guard`], so a Rust panic reaches Python as `lytk.InternalError`. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::borrow::Cow; @@ -205,6 +205,13 @@ fn check_ir_values(value: &serde_json::Value) -> Result<(), String> { Some(_) => return bad(format!("duration {v} out of range")), None => check_ir_values(v)?, }, + "lilypond_version" + if v.as_str().is_some_and(|s| { + s.parse::().is_err() + }) => + { + return bad(format!("lilypond_version {v} is not a LilyPond version")); + } // The LilyPond reader's bound (its regression tests climb to 22). "octave" if v.as_i64().is_some_and(|o| !(-128..=127).contains(&o)) => { return bad(format!("octave {v} out of range -128..=127")); @@ -459,22 +466,30 @@ fn header_dict<'py>( } /// Read LilyPond, from a file or text, releasing the GIL: the text and the -/// reading. +/// reading. With *include_paths*, includes are followed (relative to a +/// file's directory, then the paths). fn read_lilypond( py: Python<'_>, source: Result<&str, &str>, language: Option<&str>, + include_paths: Option>, ) -> PyResult<(String, adapters::ly_to_ir::LyReading)> { let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); if let Some(lang_str) = language { adapter = adapter.with_language(parse_language(lang_str)?); } + if let Some(paths) = include_paths { + adapter = adapter.with_include_paths(paths.into_iter().map(PathBuf::from).collect()); + } py.allow_threads(|| { - let text = match source { - Ok(path) => Cow::Owned(adapters::ly_to_ir::read_source(Path::new(path))?), - Err(text) => Cow::Borrowed(text), + let (text, base_dir) = match source { + Ok(path) => ( + Cow::Owned(adapters::ly_to_ir::read_source(Path::new(path))?), + Path::new(path).parent(), + ), + Err(text) => (Cow::Borrowed(text), None), }; - let reading = adapter.read_str(&text)?; + let reading = adapter.read_text(&text, base_dir)?; Ok((text.into_owned(), reading)) }) .map_err(read_err) @@ -497,15 +512,323 @@ fn first_movement( /// (fast, nothing is read); with ``semantic=True`` also what a reading /// reports: invalid durations and ratios, unknown commands, input it does not /// read. Input too large to read is an error here, not an exception. +/// *include_paths* follows includes as the readers do (see +/// :func:`from_lilypond`); an included file that cannot be read raises +/// ``OSError``. #[pyfunction] -#[pyo3(signature = (text, *, semantic=false))] -fn check_lilypond(py: Python<'_>, text: &str, semantic: bool) -> PyResult> { +#[pyo3(signature = (text, *, semantic=false, include_paths=None))] +fn check_lilypond( + py: Python<'_>, + text: &str, + semantic: bool, + include_paths: Option>, +) -> PyResult> { guard(|| { - let diagnostics = py.allow_threads(|| adapters::ly_to_ir::check(text, semantic)); + let mut adapter = adapters::ly_to_ir::LyToIrAdapter::new(); + if let Some(paths) = include_paths { + adapter = adapter.with_include_paths(paths.into_iter().map(PathBuf::from).collect()); + } + let diagnostics = py + .allow_threads(|| adapter.check_str(text, semantic)) + .map_err(read_err)?; Ok(py_diagnostics(text, &diagnostics)) }) } +// --------------------------------------------------------------------------- +// LilyPond versions +// --------------------------------------------------------------------------- + +/// A LilyPond version, as ``\\version`` states it, compared numerically: +/// ``LilyPondVersion("2.24") == LilyPondVersion("2.24.0")``. Accepted as +/// LilyPond 2.24 accepts it: ``major.minor.patch`` with an optional fourth +/// part (kept, not compared), or ``major.minor`` with an even minor (a stable +/// series). Any other string raises :class:`ParseError`. ``str()`` gives +/// ``"2.24.0"``. +#[pyclass(name = "LilyPondVersion", module = "lytk", frozen)] +#[derive(Clone)] +struct PyLilyPondVersion(adapters::ly_to_ir::LilyPondVersion); + +#[pymethods] +impl PyLilyPondVersion { + #[new] + fn new(text: &str) -> PyResult { + text.parse().map(Self).map_err(parse_error) + } + + #[getter] + fn major(&self) -> u32 { + self.0.major + } + + #[getter] + fn minor(&self) -> u32 { + self.0.minor + } + + #[getter] + fn patch(&self) -> u32 { + self.0.patch + } + + /// The fourth part (``"foo"`` in ``2.25.3.foo``), if any. + #[getter] + fn extra(&self) -> Option { + self.0.extra.clone() + } + + fn __str__(&self) -> String { + self.0.to_string() + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + let text = pyo3::types::PyString::new_bound(py, &self.0.to_string()); + Ok(format!("LilyPondVersion({})", text.repr()?)) + } + + fn __richcmp__(&self, other: PyRef<'_, Self>, op: pyo3::basic::CompareOp) -> bool { + op.matches(self.0.cmp(&other.0)) + } + + fn __hash__(&self) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + self.0.hash(&mut hasher); + hasher.finish() + } + + fn __reduce__<'py>(slf: &Bound<'py, Self>) -> (Bound<'py, PyType>, (String,)) { + (slf.get_type(), (slf.get().0.to_string(),)) + } +} + +/// The text of a version argument: a :class:`LilyPondVersion` as ``str()`` +/// writes it, or a string as given, once checked to be a version. +fn version_arg(version: &Bound<'_, PyAny>) -> PyResult { + if let Ok(v) = version.downcast::() { + return Ok(v.get().0.to_string()); + } + let text = version.extract::()?; + text.parse::() + .map_err(parse_error)?; + Ok(text) +} + +/// The version stored in `meta`, read from a ``\\version``. +fn meta_version(meta: &ir::score::ScoreMetadata) -> Option { + let version = meta.lilypond_version.as_deref()?.parse().ok()?; + Some(PyLilyPondVersion(version)) +} + +/// The version the first ``\\version`` statement of LilyPond text states, +/// from the parse tree (a commented-out one does not count): *None* when +/// there is none, or when it is not a valid version (:func:`check_lilypond` +/// reports it as ``invalid-version``). +#[pyfunction] +fn lilypond_version(py: Python<'_>, text: &str) -> PyResult> { + guard(|| { + let version = py.allow_threads(|| adapters::ly_to_ir::lilypond_version(text)); + Ok(version.map(PyLilyPondVersion)) + }) +} + +/// *text* with every ``\\version`` statement stating *version* (a +/// :class:`LilyPondVersion`, or a string, written as given), or with one added +/// at the top when there is none. +#[pyfunction] +fn set_lilypond_version( + py: Python<'_>, + text: &str, + version: &Bound<'_, PyAny>, +) -> PyResult { + let version = version_arg(version)?; + guard(|| { + py.allow_threads(|| adapters::ly_to_ir::set_lilypond_version(text, &version)) + .map_err(read_err) + }) +} + +/// *text* without its ``\\version`` statements; a statement alone on its +/// line takes the line with it. +#[pyfunction] +fn strip_lilypond_version(py: Python<'_>, text: &str) -> PyResult { + guard(|| { + py.allow_threads(|| adapters::ly_to_ir::strip_lilypond_version(text)) + .map_err(read_err) + }) +} + +// --------------------------------------------------------------------------- +// Tokens +// --------------------------------------------------------------------------- + +/// A token of LilyPond text, from :func:`tokenize`: ``kind`` is one of +/// ``"comment"``, ``"string"``, ``"command"``, ``"symbol"``, ``"number"``, +/// ``"fraction"``, ``"punctuation"``, ``"scheme"`` (the ``#`` or ``$`` that +/// starts embedded Scheme), ``"boolean"``, ``"character"``, ``"keyword"`` +/// (Scheme's) and ``"error"`` (text the grammar cannot tokenize); +/// ``text[start:end]`` is the token (character offsets); ``line`` and +/// ``column`` count from 1; ``scheme`` tells a Scheme token from a LilyPond +/// one. +#[pyclass(name = "Token", module = "lytk", frozen, get_all, eq, hash)] +#[derive(Clone, PartialEq, Eq, Hash)] +struct PyToken { + kind: &'static str, + text: String, + start: usize, + end: usize, + line: usize, + column: usize, + scheme: bool, +} + +#[pymethods] +impl PyToken { + fn __repr__(&self) -> String { + format!( + "Token({:?}, {:?}, start={}, end={}, line={}, column={}, scheme={})", + self.kind, + self.text, + self.start, + self.end, + self.line, + self.column, + if self.scheme { "True" } else { "False" } + ) + } +} + +/// The tokens of LilyPond text, in order, from the parse tree: strings and +/// comments whole, embedded Scheme as Scheme tokens (``scheme`` set), and +/// LilyPond inside it (``#{ … #}``) as LilyPond. Whitespace is no token; +/// every other character is in exactly one, so the tokens of broken input +/// cover it too (as ``"error"`` tokens where needed). +#[pyfunction] +fn tokenize(py: Python<'_>, text: &str) -> PyResult> { + guard(|| { + let tokens = py + .allow_threads(|| adapters::ly_to_ir::tokenize(text)) + .map_err(read_err)?; + let char_at = char_offsets(text, tokens.iter().flat_map(|t| [t.start, t.end])); + Ok(tokens + .into_iter() + .map(|t| PyToken { + kind: t.kind.as_str(), + text: text[t.start..t.end].to_string(), + start: char_at(t.start), + end: char_at(t.end), + line: t.line, + column: t.column, + scheme: t.scheme, + }) + .collect()) + }) +} + +/// *text* without its comments: LilyPond's (``% …``, ``%{ … %}``) and +/// embedded Scheme's (``; …``, ``#| … |#``, ``#;datum``). A block comment +/// between two tokens becomes a space, a line comment leaves its line break. +#[pyfunction] +fn strip_comments(py: Python<'_>, text: &str) -> PyResult { + guard(|| { + py.allow_threads(|| adapters::ly_to_ir::strip_comments(text)) + .map_err(read_err) + }) +} + +/// Counts of LilyPond source text, from its tokens (:func:`tokenize`): +/// ``bytes`` (UTF-8), ``lines``, ``tokens``, ``comments``, ``scheme`` +/// (embedded Scheme expressions) and ``error_tokens`` (text the grammar +/// cannot tokenize). +#[pyfunction] +fn source_stats<'py>(py: Python<'py>, text: &str) -> PyResult> { + guard(|| { + let s = py + .allow_threads(|| adapters::ly_to_ir::source_stats(text)) + .map_err(read_err)?; + let dict = PyDict::new_bound(py); + for (key, value) in [ + ("bytes", s.bytes), + ("lines", s.lines), + ("tokens", s.tokens), + ("comments", s.comments), + ("scheme", s.scheme), + ("error_tokens", s.error_tokens), + ] { + dict.set_item(key, value)?; + } + Ok(dict) + }) +} + +// --------------------------------------------------------------------------- +// Statistics +// --------------------------------------------------------------------------- + +/// A score's metadata and counts, as ``lytk info --json`` prints them: +/// ``title``, ``subtitle``, ``composer``, ``arranger``, ``lyricist``, +/// ``language``, ``lilypond_version`` (strings or *None*); ``part_count``, +/// ``note_count`` (sounding notes: chord members and grace notes each +/// count), ``voice_count`` (distinct voices of each part, summed), +/// ``bar_count``, ``duration_quarters`` (each bar as long as its longest +/// voice), ``lyric_count`` (syllables), ``chord_symbol_count``, +/// ``grace_note_count``; and ``parts``, one dict per part (``id``, ``name``, +/// ``abbreviation``, ``measures``, ``staves``, ``midi_program``, +/// ``midi_instrument``, ``voices``, ``notes``). +#[pyfunction] +fn info<'py>(py: Python<'py>, score: &PyScore) -> PyResult> { + guard(|| { + let s = &score.inner; + let counts = ir::stats::score_info(s); + let meta = &s.metadata; + let dict = PyDict::new_bound(py); + let language = meta.pitch_language.map(|l| l.as_str().to_string()); + for (key, value) in [ + ("title", &meta.title), + ("subtitle", &meta.subtitle), + ("composer", &meta.composer), + ("arranger", &meta.arranger), + ("lyricist", &meta.lyricist), + ("language", &language), + ("lilypond_version", &meta.lilypond_version), + ] { + dict.set_item(key, value)?; + } + let quarters = counts.duration_quarters; + dict.set_item("part_count", counts.parts.len())?; + dict.set_item("note_count", counts.note_count)?; + dict.set_item("voice_count", counts.voice_count)?; + dict.set_item("bar_count", counts.bar_count)?; + dict.set_item( + "duration_quarters", + *quarters.numer() as f64 / *quarters.denom() as f64, + )?; + dict.set_item("lyric_count", counts.lyric_count)?; + dict.set_item("chord_symbol_count", counts.chord_symbol_count)?; + dict.set_item("grace_note_count", counts.grace_note_count)?; + let parts = s + .parts() + .into_iter() + .zip(&counts.parts) + .map(|(part, c)| { + let d = PyDict::new_bound(py); + d.set_item("id", &part.part_id)?; + d.set_item("name", &part.name)?; + d.set_item("abbreviation", &part.abbreviation)?; + d.set_item("measures", c.measures)?; + d.set_item("staves", part.staves)?; + d.set_item("midi_program", part.midi_program)?; + d.set_item("midi_instrument", &part.midi_instrument)?; + d.set_item("voices", c.voices)?; + d.set_item("notes", c.notes)?; + Ok(d) + }) + .collect::>>()?; + dict.set_item("parts", parts)?; + Ok(dict) + }) +} + // --------------------------------------------------------------------------- // PyScore — opaque wrapper for the IR Score // --------------------------------------------------------------------------- @@ -570,6 +893,13 @@ impl PyScore { self.inner.metadata.lyricist.clone() } + /// The :class:`LilyPondVersion` the source's ``\\version`` states, for a + /// score read from LilyPond that states a valid one; else *None*. + #[getter] + fn lilypond_version(&self) -> Option { + meta_version(&self.inner.metadata) + } + /// Every header field as a dict of strings: ``title``, ``subtitle``, /// ``composer``, ``arranger`` and ``lyricist`` (LilyPond's ``poet``) when /// set, then the others (``copyright``, ``opus``, ``texidoc``, …) by key. @@ -761,6 +1091,13 @@ impl PyMusicDocument { self.inner.metadata.lyricist.clone() } + /// The :class:`LilyPondVersion` the source's ``\\version`` states (see + /// :attr:`Score.lilypond_version`). + #[getter] + fn lilypond_version(&self) -> Option { + meta_version(&self.inner.metadata) + } + /// Every header field as a dict of strings (see :attr:`Score.header`). #[getter] fn header<'py>(&self, py: Python<'py>) -> PyResult> { @@ -867,16 +1204,22 @@ fn parse_language(name: &str) -> PyResult { /// /// ``strict=True`` raises :class:`LilyPondSyntaxError` if the reading reports /// an error; the diagnostics are in :attr:`Score.diagnostics` either way. +/// With *include_paths* (a list of directories, possibly empty), +/// ``\\include`` statements are followed: relative to the file's directory (for a +/// string reader, only the paths), then the paths. An include not found is an +/// ``ignored-include`` warning; a diagnostic in an included file is reported +/// at its ``\\include``. Without it, includes are not read. #[pyfunction] -#[pyo3(signature = (path, *, language=None, strict=false))] +#[pyo3(signature = (path, *, language=None, strict=false, include_paths=None))] fn from_lilypond( py: Python<'_>, path: &str, language: Option<&str>, strict: bool, + include_paths: Option>, ) -> PyResult { guard(|| { - let (text, reading) = read_lilypond(py, Ok(path), language)?; + let (text, reading) = read_lilypond(py, Ok(path), language, include_paths)?; let (score, diagnostics) = first_movement(&text, reading, strict)?; Ok(PyScore { inner: score, @@ -889,15 +1232,43 @@ fn from_lilypond( /// block and per top-level music expression, in order, as LilyPond makes a /// score of each. Each score carries the file's diagnostics. #[pyfunction] -#[pyo3(signature = (path, *, language=None, strict=false))] +#[pyo3(signature = (path, *, language=None, strict=false, include_paths=None))] fn from_lilypond_movements( py: Python<'_>, path: &str, language: Option<&str>, strict: bool, + include_paths: Option>, +) -> PyResult> { + guard(|| { + let (text, reading) = read_lilypond(py, Ok(path), language, include_paths)?; + let diagnostics = py_diagnostics(&text, &reading.diagnostics); + check_strict(strict, &diagnostics)?; + Ok(reading + .scores + .into_iter() + .map(|inner| PyScore { + inner, + diagnostics: diagnostics.clone(), + }) + .collect()) + }) +} + +/// Parse every movement of a LilyPond string (see +/// :func:`from_lilypond_movements`). Each score carries the text's +/// diagnostics. +#[pyfunction] +#[pyo3(signature = (text, *, language=None, strict=false, include_paths=None))] +fn from_lilypond_movements_string( + py: Python<'_>, + text: &str, + language: Option<&str>, + strict: bool, + include_paths: Option>, ) -> PyResult> { guard(|| { - let (text, reading) = read_lilypond(py, Ok(path), language)?; + let (text, reading) = read_lilypond(py, Err(text), language, include_paths)?; let diagnostics = py_diagnostics(&text, &reading.diagnostics); check_strict(strict, &diagnostics)?; Ok(reading @@ -914,15 +1285,16 @@ fn from_lilypond_movements( /// Parse a LilyPond string into a :class:`Score` (its first movement); see /// :func:`from_lilypond`. #[pyfunction] -#[pyo3(signature = (text, *, language=None, strict=false))] +#[pyo3(signature = (text, *, language=None, strict=false, include_paths=None))] fn from_lilypond_string( py: Python<'_>, text: &str, language: Option<&str>, strict: bool, + include_paths: Option>, ) -> PyResult { guard(|| { - let (text, reading) = read_lilypond(py, Err(text), language)?; + let (text, reading) = read_lilypond(py, Err(text), language, include_paths)?; let (score, diagnostics) = first_movement(&text, reading, strict)?; Ok(PyScore { inner: score, @@ -936,15 +1308,16 @@ fn from_lilypond_string( /// This preserves structural information like contexts and simultaneous /// blocks. ``strict`` as in :func:`from_lilypond`. #[pyfunction] -#[pyo3(signature = (path, *, language=None, strict=false))] +#[pyo3(signature = (path, *, language=None, strict=false, include_paths=None))] fn from_lilypond_music( py: Python<'_>, path: &str, language: Option<&str>, strict: bool, + include_paths: Option>, ) -> PyResult { guard(|| { - let (text, reading) = read_lilypond(py, Ok(path), language)?; + let (text, reading) = read_lilypond(py, Ok(path), language, include_paths)?; let (score, diagnostics) = first_movement(&text, reading, strict)?; Ok(PyMusicDocument { inner: ir::lift::lift_to_music(&score), @@ -953,18 +1326,46 @@ fn from_lilypond_music( }) } +/// Parse every movement of a LilyPond file into a :class:`MusicDocument` +/// (see :func:`from_lilypond_movements`). Each carries the file's +/// diagnostics. +#[pyfunction] +#[pyo3(signature = (path, *, language=None, strict=false, include_paths=None))] +fn from_lilypond_music_movements( + py: Python<'_>, + path: &str, + language: Option<&str>, + strict: bool, + include_paths: Option>, +) -> PyResult> { + guard(|| { + let (text, reading) = read_lilypond(py, Ok(path), language, include_paths)?; + let diagnostics = py_diagnostics(&text, &reading.diagnostics); + check_strict(strict, &diagnostics)?; + Ok(reading + .scores + .iter() + .map(|score| PyMusicDocument { + inner: ir::lift::lift_to_music(score), + diagnostics: diagnostics.clone(), + }) + .collect()) + }) +} + /// Parse a LilyPond string into a :class:`MusicDocument` (Layer 1 Music /// tree); see :func:`from_lilypond_music`. #[pyfunction] -#[pyo3(signature = (text, *, language=None, strict=false))] +#[pyo3(signature = (text, *, language=None, strict=false, include_paths=None))] fn from_lilypond_music_string( py: Python<'_>, text: &str, language: Option<&str>, strict: bool, + include_paths: Option>, ) -> PyResult { guard(|| { - let (text, reading) = read_lilypond(py, Err(text), language)?; + let (text, reading) = read_lilypond(py, Err(text), language, include_paths)?; let (score, diagnostics) = first_movement(&text, reading, strict)?; Ok(PyMusicDocument { inner: ir::lift::lift_to_music(&score), @@ -974,12 +1375,21 @@ fn from_lilypond_music_string( } /// Emit a :class:`MusicDocument` as a LilyPond string. If *path* is given the -/// result is also written to that file. +/// result is also written to that file. *version* (a :class:`LilyPondVersion` +/// or a string) is the ``\\version`` written; by default ``2.24.0``. #[pyfunction] -#[pyo3(signature = (doc, path=None))] -fn to_lilypond_music(doc: &PyMusicDocument, path: Option<&str>) -> PyResult { +#[pyo3(signature = (doc, path=None, *, version=None))] +fn to_lilypond_music( + doc: &PyMusicDocument, + path: Option<&str>, + version: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let version = version.map(version_arg).transpose()?; guard(|| { - let adapter = adapters::ir_to_ly::IrToLyAdapter::new(); + let mut adapter = adapters::ir_to_ly::IrToLyAdapter::new(); + if let Some(v) = &version { + adapter = adapter.with_version(v); + } let output = adapter .convert_music(&doc.inner) .map_err(|e| PyValueError::new_err(e.to_string()))?; @@ -995,16 +1405,24 @@ fn to_lilypond_music(doc: &PyMusicDocument, path: Option<&str>) -> PyResult, language: Option<&str>, relative: Option, + version: Option<&Bound<'_, PyAny>>, ) -> PyResult { + let version = version.map(version_arg).transpose()?; guard(|| { let mut adapter = adapters::ir_to_ly::IrToLyAdapter::new(); + if let Some(v) = &version { + adapter = adapter.with_version(v); + } if let Some(lang_str) = language { adapter = adapter.with_language(parse_language(lang_str)?); } else if let Some(lang) = score.inner.metadata.pitch_language { @@ -1076,9 +1494,9 @@ fn flatten( .map(std::path::PathBuf::from) .collect(), add_markers, + ..Default::default() }; - let text = adapters::ly_flatten::flatten(Path::new(input), opts) - .map_err(|e| parse_error(e.to_string()))?; + let text = adapters::ly_flatten::flatten(Path::new(input), opts).map_err(flatten_error)?; if let Some(p) = output { std::fs::write(p, &text).map_err(|e| PyIOError::new_err(e.to_string()))?; } @@ -1086,6 +1504,45 @@ fn flatten( }) } +/// Expand the ``\\include`` directives of LilyPond *text*: relative ones +/// against *base_dir* (when given), then *include_paths*. Includes are found +/// on the parse tree (anywhere in a line, never in a comment or string). +/// Raises :class:`ParseError` for a file not found (LilyPond's own, such as +/// ``english.ly``, stay as they are), a circular include, or an expansion +/// past the bounds; ``OSError`` for a file that cannot be read. +#[pyfunction] +#[pyo3(signature = (text, *, base_dir=None, include_paths=None, add_markers=true))] +fn flatten_string( + py: Python<'_>, + text: &str, + base_dir: Option<&str>, + include_paths: Option>, + add_markers: bool, +) -> PyResult { + guard(|| { + let opts = adapters::ly_flatten::FlattenOpts { + include_paths: include_paths + .unwrap_or_default() + .into_iter() + .map(PathBuf::from) + .collect(), + add_markers, + ..Default::default() + }; + py.allow_threads(|| adapters::ly_flatten::flatten_str(text, base_dir.map(Path::new), opts)) + .map_err(flatten_error) + }) +} + +/// A flatten failure: ``OSError`` for a file that cannot be read, else +/// :class:`ParseError`. +fn flatten_error(e: adapters::ly_flatten::FlattenError) -> PyErr { + match e { + adapters::ly_flatten::FlattenError::Io { .. } => PyIOError::new_err(e.to_string()), + e => parse_error(e.to_string()), + } +} + /// Parse an ABC notation (``.abc``) file into a :class:`Score`. #[pyfunction] fn from_abc(path: &str) -> PyResult { @@ -1109,6 +1566,18 @@ fn from_abc_tunes(py: Python<'_>, path: &str) -> PyResult> { }) } +/// Parse every tune of an ABC string (see :func:`from_abc_tunes`). +#[pyfunction] +fn from_abc_tunes_string(py: Python<'_>, text: &str) -> PyResult> { + guard(|| { + let adapter = adapters::abc_to_ir::AbcToIrAdapter::new(); + let scores = py + .allow_threads(|| adapter.convert_str_tunes(text)) + .map_err(read_err)?; + Ok(scores.into_iter().map(PyScore::from).collect()) + }) +} + /// Parse an ABC notation string into a :class:`Score`. #[pyfunction] fn from_abc_string(text: &str) -> PyResult { @@ -1652,6 +2121,15 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(check_lilypond, m)?)?; m.add_class::()?; m.add_function(wrap_pyfunction!(header_fields, m)?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(lilypond_version, m)?)?; + m.add_function(wrap_pyfunction!(set_lilypond_version, m)?)?; + m.add_function(wrap_pyfunction!(strip_lilypond_version, m)?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(tokenize, m)?)?; + m.add_function(wrap_pyfunction!(strip_comments, m)?)?; + m.add_function(wrap_pyfunction!(source_stats, m)?)?; + m.add_function(wrap_pyfunction!(info, m)?)?; m.add_function(wrap_pyfunction!(_panic_for_tests, m)?)?; // Score class @@ -1668,16 +2146,20 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(from_lilypond, m)?)?; m.add_function(wrap_pyfunction!(from_lilypond_string, m)?)?; m.add_function(wrap_pyfunction!(from_lilypond_movements, m)?)?; + m.add_function(wrap_pyfunction!(from_lilypond_movements_string, m)?)?; + m.add_function(wrap_pyfunction!(from_lilypond_music_movements, m)?)?; m.add_function(wrap_pyfunction!(from_lilypond_music, m)?)?; m.add_function(wrap_pyfunction!(from_lilypond_music_string, m)?)?; m.add_function(wrap_pyfunction!(to_lilypond, m)?)?; m.add_function(wrap_pyfunction!(to_lilypond_music, m)?)?; m.add_function(wrap_pyfunction!(flatten, m)?)?; + m.add_function(wrap_pyfunction!(flatten_string, m)?)?; m.add_function(wrap_pyfunction!(to_musicxml, m)?)?; m.add_function(wrap_pyfunction!(to_mxl_bytes, m)?)?; m.add_function(wrap_pyfunction!(from_abc, m)?)?; m.add_function(wrap_pyfunction!(from_abc_string, m)?)?; m.add_function(wrap_pyfunction!(from_abc_tunes, m)?)?; + m.add_function(wrap_pyfunction!(from_abc_tunes_string, m)?)?; m.add_function(wrap_pyfunction!(from_humdrum, m)?)?; m.add_function(wrap_pyfunction!(from_humdrum_string, m)?)?; m.add_function(wrap_pyfunction!(to_humdrum, m)?)?; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 8e121d9..09d5f4c 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -23,6 +23,30 @@ use _core::ir::score::Score; // Conversion shortcuts // --------------------------------------------------------------------------- +/// Where `lytk.tokenize` loses text: the first character outside every token +/// that is no whitespace (LilyPond's, a byte-order mark included), or two +/// tokens overlapping. `None` when every other character is in exactly one +/// token. +pub fn token_gap(src: &str) -> Option { + let tokens = _core::adapters::ly_to_ir::tokenize(src).ok()?; + let blank = |c: &char| matches!(c, ' ' | '\n' | '\t' | '\r' | '\u{c}' | '\u{feff}'); + let mut at = 0; + for t in tokens + .iter() + .map(|t| (t.start, t.end)) + .chain([(src.len(), src.len())]) + { + if t.0 < at { + return Some(format!("tokens overlap at byte {}", t.0)); + } + if let Some(c) = src[at..t.0].chars().find(|c| !blank(c)) { + return Some(format!("{c:?} in no token, after byte {at}")); + } + at = t.1; + } + None +} + pub fn ly_to_score(src: &str) -> Score { LyToIrAdapter::new().convert_str(src).expect("LY → Score") } diff --git a/tests/ly_corpus.rs b/tests/ly_corpus.rs index b5598f5..36f0bb1 100644 --- a/tests/ly_corpus.rs +++ b/tests/ly_corpus.rs @@ -20,6 +20,8 @@ use _core::adapters::ly_to_ir::{check, header_fields, LyToIrAdapter}; use _core::parser::LilyPondParser; use tree_sitter::Tree; +mod common; + use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -53,6 +55,12 @@ const MUTATIONS_CAUGHT: &[(&str, usize, usize)] = &[ ("Scheme `)`", 42, 42), ]; +/// LilyPond's own tests of its errors (`expect-error = ##t`): LilyPond fails +/// them on purpose, so they are no valid files. The boards count them apart. +fn expects_error(src: &str) -> bool { + src.contains("expect-error = ##t") +} + fn read_lossy(path: &Path) -> String { let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); String::from_utf8_lossy(&bytes).into_owned() @@ -108,6 +116,8 @@ fn syntax_board() { ]; let (mut total, mut flagged_files) = (0, Vec::new()); + let (mut expected, mut caught) = (0, 0); + let mut token_gaps = Vec::new(); let mut times = Vec::new(); for (name, files) in &sets { let start = Instant::now(); @@ -121,7 +131,14 @@ fn syntax_board() { .map(|d| d.to_string()) .collect(); times.push(t.elapsed()); - if !errors.is_empty() { + if let Some(gap) = common::token_gap(&src) { + let rel = path.strip_prefix(&root).unwrap_or(path); + token_gaps.push(format!("{}: {gap}", rel.display())); + } + if expects_error(&src) { + expected += 1; + caught += usize::from(!errors.is_empty()); + } else if !errors.is_empty() { bad += 1; let rel = path.strip_prefix(&root).unwrap_or(path); flagged_files.push(format!( @@ -145,6 +162,10 @@ fn syntax_board() { "check_lilypond median: {:.3} ms", median.as_secs_f64() * 1e3 ); + println!("files expecting an error: {expected}, a syntax error found in {caught}"); + for gap in &token_gaps { + println!(" token gap: {gap}"); + } for f in &flagged_files { println!(" error: {f}"); } @@ -153,6 +174,12 @@ fn syntax_board() { "only {total} files under {}: wrong LYTK_LILYPOND_SRC?", root.display() ); + // lytk.tokenize loses no text. + assert!( + token_gaps.is_empty(), + "{} files lose text in tokens", + token_gaps.len() + ); assert!( flagged_files.len() <= SYNTAX_ERROR_FILES, "{} valid files with syntax errors, baseline {SYNTAX_ERROR_FILES}", @@ -185,6 +212,7 @@ fn reader_board() { let reader = LyToIrAdapter::new(); let start = Instant::now(); let (mut other_errors, mut refused, mut with_errors) = (0, Vec::new(), Vec::new()); + let (mut expected, mut caught) = (0, 0); // Board (d): warnings per code, as (count, files). let mut warnings: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); let mut samples: BTreeMap<&str, Vec> = BTreeMap::new(); @@ -194,13 +222,19 @@ fn reader_board() { .unwrap_or(path) .display() .to_string(); - match reader.read_str(&read_lossy(path)) { + let src = read_lossy(path); + match reader.read_str(&src) { Ok(reading) => { let errors: Vec<_> = reading .diagnostics .iter() .filter(|d| d.is_error()) .collect(); + if expects_error(&src) { + expected += 1; + caught += usize::from(!errors.is_empty()); + continue; + } if !errors.is_empty() { with_errors.push(format!("{rel}: {}", errors[0])); } @@ -230,6 +264,7 @@ fn reader_board() { refused.len(), with_errors.len() ); + println!("files expecting an error: {expected}, an error found in {caught}"); for r in &refused { println!(" refused: {r}"); } diff --git a/tests/ly_diagnostics.rs b/tests/ly_diagnostics.rs index f8c07d5..defe8d3 100644 --- a/tests/ly_diagnostics.rs +++ b/tests/ly_diagnostics.rs @@ -35,6 +35,9 @@ fn each_code_on_its_input() { ("{ \\tuplet 0/2 { c'8 d' e' } }", Error, "invalid-ratio"), ("{ c'4*1/0 }", Error, "invalid-ratio"), ("Hello world, this is not music.", Error, "not-lilypond"), + ("\\version \"2.x\"\n{ c'1 }", Error, "invalid-version"), + ("\\version \"2.25\"\n{ c'1 }", Error, "invalid-version"), + ("\\version 2.24\n{ c'1 }", Error, "invalid-version"), // Warnings: what the reader does not read. ("{ c'2048 }", Warning, "unsupported-value"), ("{ \\time 3/256 c'4 }", Warning, "unsupported-value"), diff --git a/tests/ly_source.rs b/tests/ly_source.rs new file mode 100644 index 0000000..33cca94 --- /dev/null +++ b/tests/ly_source.rs @@ -0,0 +1,111 @@ +//! The source-level LilyPond API (Epic K): `\version` (K1), tokens (K3). + +mod common; + +use _core::adapters::ir_to_ly::IrToLyAdapter; +use _core::adapters::ly_to_ir::{check, LilyPondVersion, LyToIrAdapter}; +use _core::adapters::{FromIrAdapter, ToIrAdapter}; + +#[test] +fn every_movement_carries_the_files_version() { + let src = "\\version \"2.24\"\n\\score { { c'1 } }\n{ d'1 }\n"; + let scores = LyToIrAdapter::new().convert_str_multi(src).unwrap(); + assert_eq!(scores.len(), 2); + for score in &scores { + assert_eq!(score.metadata.lilypond_version.as_deref(), Some("2.24.0")); + } + // An invalid one is an error, and no version. + let bad = "\\version \"2.x\"\n{ c'1 }"; + let score = LyToIrAdapter::new().convert_str(bad).unwrap(); + assert_eq!(score.metadata.lilypond_version, None); + let d = check(bad, false).remove(0); + assert_eq!((d.code, d.line, d.column), ("invalid-version", 1, 10)); +} + +#[test] +fn the_writer_writes_its_own_version_unless_told() { + let score = LyToIrAdapter::new() + .convert_str("\\version \"2.18.2\"\n{ c'1 }") + .unwrap(); + let out = IrToLyAdapter::new().convert(&score).unwrap(); + assert!(out.contains("\\version \"2.24.0\""), "{out}"); + let v: LilyPondVersion = "2.26".parse().unwrap(); + let out = IrToLyAdapter::new() + .with_version(&v.to_string()) + .convert(&score) + .unwrap(); + assert!(out.contains("\\version \"2.26.0\""), "{out}"); +} + +#[test] +fn tokens_cover_every_character_of_the_fixtures() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ly"); + let mut files = 0; + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let src = std::fs::read_to_string(entry.path()).unwrap(); + assert_eq!(common::token_gap(&src), None, "{}", entry.path().display()); + files += 1; + } + assert!(files >= 30); + // Broken input too: what the grammar cannot tokenize is an error token. + for src in [ + "}}} INVALID {{{", + "{ c'4 d'4", + "\\score { >> }", + "#(a (b", + "é ü ☃ ?!", + ] { + assert_eq!(common::token_gap(src), None, "{src:?}"); + } +} + +#[test] +fn includes_are_followed_with_include_paths_and_diagnostics_located() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("notes.ily"), "{ d'4 \\nosuch e'4 }\n").unwrap(); + let main = dir.path().join("main.ly"); + let src = "{ c'4 }\n\\include \"notes.ily\"\n{ f'4 c'3 }\n\\include \"gone.ily\"\n"; + std::fs::write(&main, src).unwrap(); + let located = |reading: &_core::adapters::ly_to_ir::LyReading| -> Vec<(&str, usize, String)> { + reading + .diagnostics + .iter() + .map(|d| (d.code, d.line, src[d.start..d.end].to_string())) + .collect() + }; + // Not followed by default: two warnings, two movements. + let plain = LyToIrAdapter::new().read_file(&main).unwrap(); + assert_eq!(plain.scores.len(), 2); + let codes: Vec<&str> = plain.diagnostics.iter().map(|d| d.code).collect(); + assert_eq!( + codes, + ["ignored-include", "invalid-duration", "ignored-include"] + ); + // Followed: the included music is a movement; what is wrong in it is + // reported at its \include, the rest where it stands in the source. + let reading = LyToIrAdapter::new() + .with_include_paths(vec![]) + .read_file(&main) + .unwrap(); + assert_eq!(reading.scores.len(), 3); + assert_eq!( + located(&reading), + [ + ("unknown-command", 2, "\\include \"notes.ily\"".to_string()), + ("invalid-duration", 3, "3".to_string()), + ("ignored-include", 4, "\\include".to_string()), + ] + ); + assert!(reading.diagnostics[0] + .message + .starts_with("in `notes.ily`: unknown command")); + assert!(reading.diagnostics[2] + .message + .contains("no such file in the include paths")); + // The check follows them too. + let checked = LyToIrAdapter::new() + .with_include_paths(vec![dir.path().to_path_buf()]) + .check_str(src, true) + .unwrap(); + assert_eq!(checked.len(), 3); +} diff --git a/tests/test_bindings.py b/tests/test_bindings.py index 62a2df1..1c2766c 100644 --- a/tests/test_bindings.py +++ b/tests/test_bindings.py @@ -460,12 +460,180 @@ def test_header_fields_locate_each_assignment(self): cut = text[: fields[0].start] + text[fields[0].end :] assert [f.key for f in lytk.header_fields(cut)] == ["piece"] + def test_movements_of_a_string(self): + text = "\\version \"2.24.0\"\n{ c'1 }\n\\score { { d'1 e'1 } }\n{ c'3 }\n" + scores = lytk.from_lilypond_movements_string(text) + assert [len(s.notes()) for s in scores] == [1, 2, 1] + assert all([d.code for d in s.diagnostics] == ["invalid-duration"] for s in scores) + with pytest.raises(lytk.LilyPondSyntaxError): + lytk.from_lilypond_movements_string(text, strict=True) + + def test_music_movements(self, tmp_path): + path = tmp_path / "two.ly" + path.write_text('\\version "2.24.0"\n{ c\'1 }\n\\score { { d\'1 e\'1 } }\n') + docs = lytk.from_lilypond_music_movements(str(path)) + assert [len(d.notes()) for d in docs] == [1, 2] + assert all(d.lilypond_version == lytk.LilyPondVersion("2.24") for d in docs) + def test_language_files_set_the_pitch_names(self): score = lytk.from_lilypond_string('\\include "english.ly"\n{ cs\'4 }') assert [n[2] for n in score.notes()] == [61] assert score.diagnostics == [] +class TestLilyPondVersion: + """\\version read, compared and edited (Epic K1).""" + + def test_versions_compare_numerically(self): + v = lytk.LilyPondVersion + assert v("2.24") == v("2.24.0") and hash(v("2.24")) == hash(v("2.24.0")) + assert v("2.24.10") > v("2.24.9") > v("2.22.2") + assert sorted([v("2.26.0"), v("2.24.0")]) == [v("2.24.0"), v("2.26.0")] + assert (str(v("2.24")), repr(v("2.24"))) == ("2.24.0", "LilyPondVersion('2.24.0')") + assert v("2.25.3.x").extra == "x" + import pickle + + assert pickle.loads(pickle.dumps(v("2.26.0"))) == v("2.26.0") + for bad in ("2.25", "two", "2.24.0.a.b"): + with pytest.raises(lytk.ParseError): + v(bad) + + def test_version_of_text_and_of_a_score(self): + text = '% \\version "1.0.0"\n\\version "2.24"\n{ c\'1 }' + assert lytk.lilypond_version(text) == lytk.LilyPondVersion("2.24.0") + assert lytk.lilypond_version("{ c'1 }") is None + assert lytk.from_lilypond_string(text).lilypond_version == lytk.LilyPondVersion("2.24.0") + assert lytk.from_lilypond_music_string(text).lilypond_version.minor == 24 + assert lytk.from_lilypond_string("{ c'1 }").lilypond_version is None + score = lytk.from_lilypond_string(text) + assert lytk.Score.from_json(score.to_json()).lilypond_version == score.lilypond_version + + def test_set_and_strip(self): + text = '\\version "2.18.2"\n{ c\'1 }\n' + assert lytk.set_lilypond_version(text, "2.24") == '\\version "2.24"\n{ c\'1 }\n' + assert lytk.set_lilypond_version("{ c'1 }", lytk.LilyPondVersion("2.26.0")).startswith('\\version "2.26.0"\n') + assert lytk.strip_lilypond_version(text) == "{ c'1 }\n" + with pytest.raises(lytk.ParseError): + lytk.set_lilypond_version(text, "2.25") + + def test_writers_take_the_version(self): + score = lytk.from_lilypond_string("{ c'1 }") + assert '\\version "2.24.0"' in lytk.to_lilypond(score) + assert '\\version "2.26"' in lytk.to_lilypond(score, version="2.26") + doc = lytk.from_lilypond_music_string("{ c'1 }") + assert '\\version "2.26.0"' in lytk.to_lilypond_music(doc, version=lytk.LilyPondVersion("2.26.0")) + with pytest.raises(lytk.ParseError): + lytk.to_lilypond(score, version="latest") + + def test_an_invalid_version_is_an_error(self): + codes = [d.code for d in lytk.check_lilypond('\\version "2.x"\n{ c\'1 }')] + assert codes == ["invalid-version"] + + +class TestTokens: + """Tokens and comment stripping (Epic K3).""" + + def test_tokens_cover_the_text(self): + text = "% é\n\\relative c'' { \\time 3/4 c4 \"a \\\"b\" #(x 1) }" + tokens = lytk.tokenize(text) + assert [(t.kind, t.text) for t in tokens][:4] == [ + ("comment", "% é"), + ("command", "\\relative"), + ("symbol", "c"), + ("punctuation", "'"), + ] + assert {t.kind for t in tokens} >= {"fraction", "number", "string", "scheme"} + assert [(t.kind, t.text) for t in tokens if t.scheme] == [ + ("scheme", "#"), + ("punctuation", "("), + ("symbol", "x"), + ("number", "1"), + ("punctuation", ")"), + ] + for t in tokens: + assert text[t.start : t.end] == t.text # character offsets + assert (tokens[1].line, tokens[1].column) == (2, 1) + leftover = text + for t in reversed(tokens): + leftover = leftover[: t.start] + leftover[t.end :] + assert leftover.strip() == "" + assert [t.kind for t in lytk.tokenize("}}} ?!")].count("punctuation") >= 3 + + def test_strip_comments(self): + assert lytk.strip_comments("c4%{x%}d4 % tail\nr4") == "c4 d4 \nr4" + assert lytk.strip_comments("#(a ; s\n b)") == "#(a \n b)" + + +class TestStatistics: + """lytk.info and lytk.source_stats (Epic K4).""" + + def test_counts(self): + score = lytk.from_lilypond_string( + "\\version \"2.24.0\"\n{ \\grace d''8 e''4 f'' a'' | c''1 }\n\\addlyrics { la la la la la }" + ) + data = lytk.info(score) + assert data["lilypond_version"] == "2.24.0" + assert (data["part_count"], data["note_count"], data["grace_note_count"]) == (1, 7, 1) + assert (data["bar_count"], data["duration_quarters"], data["lyric_count"]) == (2, 8.0, 5) + assert data["parts"][0]["notes"] == 7 and data["parts"][0]["measures"] == 2 + two = lytk.info(lytk.from_lilypond_string("<< \\new ChordNames \\chordmode { c1 g1 } \\new Staff << { c''1 d''1 } \\\\ { c'1 d'1 } >> >>")) + assert (two["voice_count"], two["chord_symbol_count"], two["note_count"]) == (2, 2, 4) + + def test_source_stats(self): + stats = lytk.source_stats("% a\n#(define x 1)\n{ c'4 %{b%} }\n") + assert stats == {"bytes": 32, "lines": 3, "tokens": 13, "comments": 2, "scheme": 1, "error_tokens": 0} + + +class TestIncludes: + """flatten_string and include_paths on the readers (Epic K2).""" + + def test_flatten_string(self, tmp_path): + (tmp_path / "part.ily").write_text("d'4\n") + text = "%{ x %}\n{ c'4 \\include \"part.ily\" e'4 }\n\\include \"english.ly\"\n" + flat = lytk.flatten_string(text, base_dir=str(tmp_path), add_markers=False) + assert flat == "%{ x %}\n{ c'4 \nd'4\n e'4 }\n\\include \"english.ly\"\n" + assert lytk.flatten_string(text, include_paths=[str(tmp_path)]).count("BEGIN INCLUDE") == 1 + with pytest.raises(lytk.ParseError): + lytk.flatten_string('\\include "gone.ily"') + + def test_readers_follow_includes_when_given_paths(self, tmp_path): + (tmp_path / "notes.ily").write_text("{ d'4 \\nosuch e'4 }\n") + main = tmp_path / "main.ly" + main.write_text('\\include "notes.ily"\n{ c\'3 }\n') + assert len(lytk.from_lilypond_movements(str(main))) == 1 + followed = lytk.from_lilypond_movements(str(main), include_paths=[]) + assert len(followed) == 2 + first = lytk.from_lilypond(str(main), include_paths=[]) + assert [(d.code, d.line) for d in first.diagnostics] == [("unknown-command", 1), ("dropped-music", 2), ("invalid-duration", 2)] + assert first.diagnostics[0].message.startswith("in `notes.ily`") + text = main.read_text() + codes = [d.code for d in lytk.check_lilypond(text, semantic=True, include_paths=[str(tmp_path)])] + assert codes == ["unknown-command", "invalid-duration"] + score = lytk.from_lilypond_string(text, include_paths=[str(tmp_path)]) + assert len(score.notes()) == 2 + + +class TestPythonApiReference: + """docs/python-api.md is generated from the stubs (Epic L4).""" + + def test_the_reference_is_current(self): + import importlib.util + + spec = importlib.util.spec_from_file_location("python_api", "scripts/python_api.py") + generator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generator) + current = Path("docs/python-api.md").read_text(encoding="utf-8") + assert current == generator.render(), "docs/python-api.md is stale: run python scripts/python_api.py" + + def test_the_stubs_cover_the_extension(self): + import ast + + stub = ast.parse(Path("src/lytk/_core.pyi").read_text(encoding="utf-8")) + stubbed = {n.name for n in stub.body if isinstance(n, (ast.FunctionDef, ast.ClassDef))} + public = {n for n in dir(lytk._core) if not n.startswith("_") and callable(getattr(lytk._core, n))} + assert public <= stubbed, f"no stub for {sorted(public - stubbed)}" + + class TestPanicFirewall: """A Rust panic reaches Python as lytk.InternalError, never as pyo3's PanicException (a BaseException), and prints nothing.""" diff --git a/tests/test_cli.py b/tests/test_cli.py index ba7a695..dd8150a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -399,6 +399,7 @@ def test_info_json(self): assert data["part_count"] == 1 assert data["note_count"] == len(lytk.from_musicxml(str(FIXTURE_XML)).iter_parts()[0].notes) assert data["parts"][0]["measures"] == 28 + assert data == lytk.info(lytk.from_musicxml(str(FIXTURE_XML))) def test_positions(self): data = json.loads(ok("positions", str(FIXTURE_XML)).stdout) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 9a2cbae..dbd63f1 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -7,10 +7,12 @@ import numpy as np import pytest +import lytk from lytk.datasets import ( SUPPORTED_EXTENSIONS, Dataset, FolderDataset, + RecordsDataset, load_document, ) @@ -395,3 +397,211 @@ def test_agrees_with_pytorch(self, tmp_path): fp, fl = next(iter(ds.to_tensorflow_dataloader("event_sequence", batch_size=3))) assert fl.numpy().tolist() == tl.tolist() assert np.array_equal(fp.numpy(), tp.numpy()) + + +# --------------------------------------------------------------------------- +# Epic L: errors, movements, cache keys, ids, records, groups +# --------------------------------------------------------------------------- + + +def _write(folder: Path, files: dict[str, str | bytes]) -> Path: + folder.mkdir(parents=True, exist_ok=True) + for name, content in files.items(): + path = folder / name + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, bytes): + path.write_bytes(content) + else: + path.write_text(content) + return folder + + +def _pitches(doc) -> list[int]: + return [int(p) for p in lytk.to_note_array(doc, 24)[:, 2]] + + +class TestOnError: + FILES = {"a.ly": "{ c'4 d' }\n", "b.ly": b"{ c'4 \xe9 }\n", "c.ly": "{ e'1 }\n"} + + def test_raise_is_the_default(self, tmp_path): + ds = FolderDataset(_write(tmp_path / "f", self.FILES)) + with pytest.raises(lytk.ParseError): + list(ds) + + def test_skip_records_the_error(self, tmp_path): + ds = FolderDataset(_write(tmp_path / "f", self.FILES), on_error="skip") + assert [_pitches(d) for d in ds] == [[60, 62], [64]] + assert list(ds.errors) == ["b.ly"] and ds.errors["b.ly"].startswith("ParseError") + assert len(ds.to_note_arrays(resolution=24)) == 2 + assert len(ds.metrics()) == 2 + with pytest.raises(lytk.ParseError): + ds[1] # indexing asks for that item + # A subset records into its parent's errors. + ds.errors.clear() + (whole,) = ds.split((1.0,)) + assert len(list(whole)) == 2 and list(ds.errors) == ["b.ly"] + + def test_warn(self, tmp_path): + ds = FolderDataset(_write(tmp_path / "f", self.FILES), on_error="warn") + with pytest.warns(RuntimeWarning, match="b.ly"): + assert len(list(ds)) == 2 + + def test_strict_errors_are_skipped_too(self, tmp_path): + folder = _write(tmp_path / "f", {"a.ly": "{ c'4 }\n", "b.ly": "{ c'4 d'4\n"}) + ds = FolderDataset(folder, strict=True, on_error="skip") + assert len(list(ds)) == 1 and list(ds.errors) == ["b.ly"] + + def test_bad_choices(self, tmp_path): + with pytest.raises(ValueError): + FolderDataset(tmp_path, on_error="ignore") + with pytest.raises(ValueError): + FolderDataset(tmp_path, movements="some") + + +class TestMovementsAndReadingOptions: + def test_each_movement_is_an_item(self, tmp_path): + folder = _write( + tmp_path / "f", + { + "b.ly": "\\score { { c'1 } }\n\\score { { d'1 } }\n", + "a.abc": "X:1\nT:A\nK:C\nC4|\n\nX:2\nT:B\nK:C\nD4|\n", + }, + ) + ds = FolderDataset(folder, movements="all") + assert ds.ids == ["a.abc#1", "a.abc#2", "b.ly#1", "b.ly#2"] + assert [_pitches(d) for d in ds] == [[60], [62], [60], [62]] + assert len(FolderDataset(folder)) == 2 + + def test_lilypond_options_pass_through(self, tmp_path): + lib = _write(tmp_path / "lib", {"notes.ily": "{ e'4 }\n"}) + folder = _write(tmp_path / "f", {"a.ly": '\\include "notes.ily"\n{ cs\'4 }\n'}) + plain = FolderDataset(folder, movements="all") + assert [_pitches(d) for d in plain] == [[]] # `cs` is no Dutch note name + ds = FolderDataset(folder, movements="all", include_paths=[lib], language="english") + assert [_pitches(d) for d in ds] == [[64], [61]] + + def test_quantize_reaches_the_midi_reader(self): + ds = FolderDataset(MIDI_DIR, recursive=False, quantize=16) + assert len(ds) > 0 and hasattr(ds[0], "to_score") + + +class TestCacheKeys: + def test_defaults_share_entries_and_writes_are_atomic(self, tmp_path): + cache = tmp_path / "cache" + ds = FolderDataset(_small_ly_folder(tmp_path, 2), cache_dir=cache) + ds.to_note_arrays() + list(ds.iter_representation("note_array")) + assert len(list(cache.glob("*.npy"))) == 2 + assert not list(cache.glob("*.tmp")) + + def test_changed_content_is_not_served_stale(self, tmp_path): + folder = _write(tmp_path / "f", {"a.ly": "{ c'4 }\n"}) + ds = FolderDataset(folder, cache_dir=tmp_path / "cache") + first = ds.to_note_arrays(resolution=24)[0] + (folder / "a.ly").write_text("{ g'4 a'4 }\n") + second = ds.to_note_arrays(resolution=24)[0] + assert first.shape[0] == 1 and second.shape[0] == 2 + + def test_subsets_use_the_cache(self, tmp_path): + cache = tmp_path / "cache" + ds = FolderDataset(_small_ly_folder(tmp_path, 3), cache_dir=cache) + train, rest = ds.split((2, 1), seed=0) + train.to_note_arrays(resolution=24) + assert len(list(cache.glob("*.npy"))) == 2 + + +class TestIds: + def test_folder_ids_are_relative_paths(self, tmp_path): + folder = _write(tmp_path / "f", {"x/a.ly": "{ c'1 }\n", "y/a.ly": "{ d'1 }\n"}) + ds = FolderDataset(folder) + assert ds.ids == ["x/a.ly", "y/a.ly"] + first, second = ds.split((1, 1), seed=3) + assert sorted(first.ids + second.ids) == ds.ids + + def test_groups_stay_together(self, tmp_path): + files = {f"{g}/{i}.ly": "{ c'1 }\n" for g in "abcdefgh" for i in range(3)} + ds = FolderDataset(_write(tmp_path / "f", files)) + parts = ds.split((0.5, 0.25, 0.25), seed=1, groups=lambda item_id: item_id.split("/")[0]) + seen = [{i.split("/")[0] for i in part.ids} for part in parts] + assert sum(len(s) for s in seen) == 8 # no folder in two parts + assert [len(p) for p in parts] == [12, 6, 6] + # Deterministic. + again = ds.split((0.5, 0.25, 0.25), seed=1, groups=lambda item_id: item_id.split("/")[0]) + assert [p.ids for p in parts] == [p.ids for p in again] + + +class TestRecordsDataset: + RECORDS = [ + {"id": "r1", "text": "{ c'4 }", "split": "train", "cluster": "k1"}, + {"id": "r2", "text": "{ d'4 }", "split": "train", "cluster": "k1"}, + {"id": "r3", "text": "{ e'4 }", "split": "test", "cluster": "k2"}, + {"id": "r4", "text": "\\score { { f'4 } }\n\\score { { g'4 } }", "split": "valid", "cluster": "k3"}, + ] + + def test_from_jsonl_keeps_ids_and_records(self, tmp_path): + import json + + path = tmp_path / "scores.jsonl" + path.write_text("".join(json.dumps(r) + "\n" for r in self.RECORDS) + "\n") + ds = RecordsDataset.from_jsonl(path) + assert ds.ids == ["r1", "r2", "r3", "r4"] + assert ds.record(2)["cluster"] == "k2" + assert [_pitches(d) for d in ds] == [[60], [62], [64], [65]] + every = RecordsDataset.from_records(self.RECORDS, movements="all") + assert every.ids[-2:] == ["r4#1", "r4#2"] + assert every.record(4)["id"] == "r4" + assert _pitches(every[4]) == [67] + + def test_own_splits_are_kept(self): + ds = RecordsDataset.from_records(self.RECORDS, split_field="split") + parts = ds.split() + assert {k: v.ids for k, v in parts.items()} == { + "train": ["r1", "r2"], + "test": ["r3"], + "valid": ["r4"], + } + assert list(ds.split(field="cluster")) == ["k1", "k2", "k3"] + with pytest.raises(ValueError): + ds.split((0.5, 0.5), field="split") + # Ratios are a deliberate re-split. + a, b = ds.split((0.5, 0.5), seed=0, groups="cluster") + assert {r for p in (a, b) for r in p.ids} == {"r1", "r2", "r3", "r4"} + assert {"r1", "r2"} <= set(a.ids) or {"r1", "r2"} <= set(b.ids) + + def test_other_text_formats(self): + abc = {"id": "t", "text": "X:1\nK:C\nC4|\n\nX:2\nK:C\nE4|\n"} + ds = RecordsDataset.from_records([abc], format="abc", movements="all") + assert [_pitches(d) for d in ds] == [[60], [64]] + with pytest.raises(ValueError): + RecordsDataset.from_records([abc], format="midi") + + def test_bad_records(self): + with pytest.raises(ValueError, match="duplicate"): + RecordsDataset.from_records([{"id": 1, "text": "{ c'1 }"}, {"id": 1, "text": "{ d'1 }"}]) + with pytest.raises(ValueError, match="text"): + RecordsDataset.from_records([{"id": 1}]) + + def test_errors_and_cache(self, tmp_path): + records = [{"id": "ok", "text": "{ c'4 }"}, {"id": "bad", "text": "{ c'4"}] + ds = RecordsDataset.from_records(records, strict=True, on_error="skip", cache_dir=tmp_path / "c") + assert len(ds.to_note_arrays()) == 1 and list(ds.errors) == ["bad"] + assert len(list((tmp_path / "c").glob("*.npy"))) == 1 + + +class TestReturnIds: + def test_pytorch(self, tmp_path): + pytest.importorskip("torch") + ds = FolderDataset(_small_ly_folder(tmp_path, 3)) + tensor, item_id = ds.to_pytorch_dataset("note_array", return_ids=True, resolution=24)[0] + assert item_id == ds.ids[0] and tensor.ndim == 2 + loader = ds.to_pytorch_dataloader("note_array", batch_size=3, return_ids=True) + padded, lengths, ids = next(iter(loader)) + assert ids == ds.ids and padded.shape[0] == 3 and lengths.shape == (3,) + + def test_tensorflow(self, tmp_path): + pytest.importorskip("tensorflow") + ds = FolderDataset(_small_ly_folder(tmp_path, 2)) + items = list(ds.to_tensorflow_dataset("note_array", return_ids=True, resolution=24)) + assert [i.numpy().decode() for _, i in items] == ds.ids + padded, lengths, ids = next(iter(ds.to_tensorflow_dataloader("note_array", batch_size=2, return_ids=True))) + assert [i.decode() for i in ids.numpy()] == ds.ids and padded.shape[0] == 2