diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65afebf..e51948d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,20 @@ and `MarkupString.Pueblo`. The packages share one version and are released toget
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project
follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-## 2.20.0 — Unreleased
+## 2.21.0 — Unreleased
+
+### Added
+
+- **Data markup.** `IDataMarkup` marks a layer that says what the text is rather than how it looks,
+ such as a list separator. No format draws it: the renderer takes it off each run before any emitter
+ sees the run, so the text is written as the layers around it write it. It is stored, carried and
+ compared like any other layer. See [Data markup](docs/custom-markup.md#data-markup).
+- **`MarkupText.WithoutVisualMarkup()`**: the plain text with every data layer kept, for stripping
+ colour from text that also carries data.
+- **`MarkupText.WithoutDataMarkup()`**: the text with every data layer taken off and its visual
+ markup and points kept, for text whose data must not be trusted.
+
+## 2.20.0
### Added
diff --git a/MarkupString.Tests/DataMarkupTests.cs b/MarkupString.Tests/DataMarkupTests.cs
new file mode 100644
index 0000000..5dbe7fb
--- /dev/null
+++ b/MarkupString.Tests/DataMarkupTests.cs
@@ -0,0 +1,168 @@
+using System.Text.Json;
+using MarkupString.Ansi;
+using MarkupString.Html;
+namespace MarkupString.Tests;
+
+///
+/// Data markup says what text is, not how it looks: no format draws it, taking the visual markup off keeps
+/// it, and it is carried and stored like any other layer.
+///
+public class DataMarkupTests
+{
+ private sealed record Separator : IDataMarkup;
+
+ private sealed class SeparatorCodec : IMarkupCodec
+ {
+ public string Kind => "test-separator";
+ public Type MarkupType => typeof(Separator);
+ public void Write(Utf8JsonWriter writer, IMarkup markup) { }
+ public IMarkup Read(JsonElement element) => new Separator();
+ }
+
+ /// Draws a separator as {|}, to show that the renderer never calls an emitter for data.
+ private sealed class SeparatorEmitter : IMarkupEmitter
+ {
+ public Type MarkupType => typeof(Separator);
+ public MarkupFormat Format => MarkupFormat.Ansi;
+ public void Emit(IMarkup markup, ReadOnlySpan body, in EmitContext context, System.Buffers.IBufferWriter output)
+ {
+ output.Write("{");
+ output.Write(body);
+ output.Write("}");
+ }
+ }
+
+ private static readonly MarkupRegistry Registry = MarkupRegistry.Empty.WithAnsi().WithHtml().With(new SeparatorCodec());
+
+ private static readonly AnsiMarkup Red = AnsiMarkup.Create(foreground: new AnsiColor.Standard(1, false));
+
+ private static MarkupText Marked(string text) => MarkupText.Wrap(new Separator(), text);
+
+ private static bool IsData(MarkupText text, int index) =>
+ text.Runs.Any(run => index >= run.Start && index < run.End && run.Markups.Any(markup => markup is IDataMarkup));
+
+ [Test]
+ public async Task DataIsWrittenAsItsText()
+ {
+ var text = MarkupText.Concat([MarkupText.Plain("a"), Marked("|"), MarkupText.Plain("b")]);
+
+ await Assert.That(text.Render(MarkupFormat.Ansi, Registry)).IsEqualTo("a|b");
+ await Assert.That(text.Render(MarkupFormat.Html, Registry)).IsEqualTo("a|b");
+ await Assert.That(text.Render(MarkupFormat.Plain, Registry)).IsEqualTo("a|b");
+ }
+
+ [Test]
+ public async Task DataUnderColourIsWrittenInTheColour()
+ {
+ var coloured = MarkupText.Concat([MarkupText.Plain("a"), MarkupText.Wrap(Red, Marked("|")), MarkupText.Plain("b")]);
+ var colourOnly = MarkupText.Concat([MarkupText.Plain("a"), MarkupText.Wrap(Red, "|"), MarkupText.Plain("b")]);
+
+ await Assert.That(coloured.Render(MarkupFormat.Ansi, Registry)).IsEqualTo(colourOnly.Render(MarkupFormat.Ansi, Registry));
+ await Assert.That(coloured.Render(MarkupFormat.Html, Registry)).IsEqualTo(colourOnly.Render(MarkupFormat.Html, Registry));
+ }
+
+ [Test]
+ public async Task AnEmitterForDataIsNeverCalled()
+ {
+ var registry = Registry.With(new SeparatorEmitter());
+
+ await Assert.That(MarkupText.Concat([MarkupText.Plain("a"), Marked("|")]).Render(MarkupFormat.Ansi, registry)).IsEqualTo("a|");
+ }
+
+ [Test]
+ public async Task WithoutVisualMarkup_KeepsTheDataAndDropsTheColour()
+ {
+ var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Wrap(Red, Marked("|")), MarkupText.Plain("b"), Marked("|")]);
+
+ var stripped = text.WithoutVisualMarkup();
+
+ await Assert.That(stripped.Text).IsEqualTo("a|b|");
+ await Assert.That(stripped.Runs.Length).IsEqualTo(2);
+ await Assert.That(stripped.Runs.All(run => run.Markups.Count == 1 && run.Markups[0] is Separator)).IsTrue();
+ await Assert.That(IsData(stripped, 1)).IsTrue();
+ await Assert.That(IsData(stripped, 3)).IsTrue();
+ await Assert.That(IsData(stripped, 0)).IsFalse();
+ }
+
+ [Test]
+ public async Task WithoutVisualMarkup_JoinsDataThatColourHadSplit()
+ {
+ var text = MarkupText.Concat([MarkupText.Wrap(Red, Marked("|")), Marked("|")]);
+
+ var stripped = text.WithoutVisualMarkup();
+
+ await Assert.That(stripped.Runs.Length).IsEqualTo(1);
+ await Assert.That(stripped.Runs[0].Length).IsEqualTo(2);
+ }
+
+ [Test]
+ public async Task WithoutVisualMarkup_WithNoData_IsThePlainText()
+ {
+ var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Bell(), MarkupText.Plain("b")]);
+
+ var stripped = text.WithoutVisualMarkup();
+
+ await Assert.That(stripped.Text).IsEqualTo("ab");
+ await Assert.That(stripped.Runs.IsDefaultOrEmpty).IsTrue();
+ }
+
+ [Test]
+ public async Task WithoutVisualMarkup_DropsPointsWithTheirCarriers()
+ {
+ var text = MarkupText.Concat([Marked("|"), MarkupText.Bell(), MarkupText.Plain("b")]);
+
+ var stripped = text.WithoutVisualMarkup();
+
+ await Assert.That(stripped.Text).IsEqualTo("|b");
+ await Assert.That(IsData(stripped, 0)).IsTrue();
+ }
+
+ [Test]
+ public async Task WithoutDataMarkup_KeepsTheColourAndDropsTheData()
+ {
+ var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Wrap(Red, Marked("|")), MarkupText.Plain("b"), Marked("|")]);
+
+ var unmarked = text.WithoutDataMarkup();
+
+ await Assert.That(unmarked.Text).IsEqualTo("a|b|");
+ await Assert.That(IsData(unmarked, 1)).IsFalse();
+ await Assert.That(IsData(unmarked, 3)).IsFalse();
+ await Assert.That(unmarked.Render(MarkupFormat.Ansi, Registry))
+ .IsEqualTo(MarkupText.Concat([MarkupText.Wrap(Red, "a|"), MarkupText.Plain("b|")]).Render(MarkupFormat.Ansi, Registry));
+ }
+
+ [Test]
+ public async Task WithoutDataMarkup_KeepsPoints()
+ {
+ var text = MarkupText.Concat([Marked("|"), MarkupText.Bell(), MarkupText.Plain("b")]);
+
+ var unmarked = text.WithoutDataMarkup();
+
+ await Assert.That(unmarked.Text).IsEqualTo(text.Text);
+ await Assert.That(unmarked.ToPlainText()).IsEqualTo("|b");
+ await Assert.That(IsData(unmarked, 0)).IsFalse();
+ }
+
+ [Test]
+ public async Task WithoutDataMarkup_WithNoData_IsTheSameText()
+ {
+ var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Plain("b")]);
+
+ await Assert.That(ReferenceEquals(text.WithoutDataMarkup(), text)).IsTrue();
+ }
+
+ [Test]
+ public async Task DataIsIgnoredByEquality()
+ => await Assert.That(MarkupText.Concat([MarkupText.Plain("a"), Marked("|")])).IsEqualTo(MarkupText.Plain("a|"));
+
+ [Test]
+ public async Task DataIsStoredAndReadBack()
+ {
+ var text = MarkupText.Concat([MarkupText.Plain("a"), Marked("|"), MarkupText.Plain("b")]);
+
+ var back = MarkupTextSerializer.Deserialize(MarkupTextSerializer.Serialize(text, Registry), Registry);
+
+ await Assert.That(IsData(back, 1)).IsTrue();
+ await Assert.That(IsData(back, 0)).IsFalse();
+ }
+}
diff --git a/MarkupString/IDataMarkup.cs b/MarkupString/IDataMarkup.cs
new file mode 100644
index 0000000..3421511
--- /dev/null
+++ b/MarkupString/IDataMarkup.cs
@@ -0,0 +1,16 @@
+namespace MarkupString;
+
+///
+/// A markup that says what the text it covers is, rather than how it looks: that a character is a list
+/// separator, say, and not part of an item.
+///
+///
+/// No format writes anything for a data layer. The renderer leaves it out before any emitter sees the
+/// run, so the text it covers is written as the layers around it would write it, and an emitter registered
+/// for a data type is never called.
+/// Data stays where visual markup goes: takes every other
+/// layer off and keeps these. Like every layer, it is ignored by and
+/// equality, carried through slicing, concatenation and joining, and stored through its codec
+/// (); a reader without the codec keeps it as an .
+///
+public interface IDataMarkup : IMarkup;
diff --git a/MarkupString/MarkupText.cs b/MarkupString/MarkupText.cs
index 252858b..f276449 100644
--- a/MarkupString/MarkupText.cs
+++ b/MarkupString/MarkupText.cs
@@ -154,6 +154,78 @@ public static MarkupText Join(Func separator, IEnumerable
public string ToPlainText() => _plainText ??= WithoutPoints();
+ ///
+ /// The text with every visual layer taken off and its data layers () kept: what
+ /// gives, plus the data. Points go with their carriers, as they do there.
+ ///
+ public MarkupText WithoutVisualMarkup()
+ {
+ if (Runs.IsDefaultOrEmpty) return this;
+
+ var hasData = false;
+ foreach (var run in Runs)
+ {
+ foreach (var markup in run.Markups)
+ {
+ if (markup is not IDataMarkup) continue;
+ hasData = true;
+ break;
+ }
+ if (hasData) break;
+ }
+ if (!hasData) return Plain(ToPlainText());
+
+ var parts = new List(Runs.Length * 2 + 1);
+ var position = 0;
+ foreach (var run in Runs)
+ {
+ if (run.Start > position) parts.Add(Plain(Text[position..run.Start]));
+ position = run.End;
+ if (IsPoint(run)) continue;
+
+ var data = new List();
+ foreach (var markup in run.Markups)
+ if (markup is IDataMarkup) data.Add(markup);
+ var body = Text.Substring(run.Start, run.Length);
+ parts.Add(data.Count == 0 ? Plain(body) : Wrap(MarkupSet.Of(data), body));
+ }
+ if (position < Text.Length) parts.Add(Plain(Text[position..]));
+ return Concat(parts);
+ }
+
+ ///
+ /// The text with every data layer () taken off and everything else kept: the
+ /// counterpart of , for text whose data must not be trusted.
+ ///
+ public MarkupText WithoutDataMarkup()
+ {
+ if (Runs.IsDefaultOrEmpty) return this;
+
+ var hasData = false;
+ foreach (var run in Runs)
+ {
+ foreach (var markup in run.Markups)
+ {
+ if (markup is not IDataMarkup) continue;
+ hasData = true;
+ break;
+ }
+ if (hasData) break;
+ }
+ if (!hasData) return this;
+
+ var builder = ImmutableArray.CreateBuilder(Runs.Length);
+ foreach (var run in Runs)
+ {
+ var kept = new List(run.Markups.Count);
+ foreach (var markup in run.Markups)
+ if (markup is not IDataMarkup) kept.Add(markup);
+ if (kept.Count == run.Markups.Count) builder.Add(run);
+ else if (kept.Count > 0) builder.Add(run with { Markups = MarkupSet.Of(kept) });
+ }
+ return new MarkupText(Text, builder.ToImmutable());
+ }
+
/// The same as .
public override string ToString() => ToPlainText();
public bool Equals(MarkupText? other) => other is not null && string.Equals(ToPlainText(), other.ToPlainText(), StringComparison.Ordinal);
diff --git a/MarkupString/MarkupTextRenderer.cs b/MarkupString/MarkupTextRenderer.cs
index 6d0b4ce..f12908a 100644
--- a/MarkupString/MarkupTextRenderer.cs
+++ b/MarkupString/MarkupTextRenderer.cs
@@ -379,7 +379,13 @@ private static bool RenderRun(
IBufferWriter output)
{
var body = content.Slice(run.Start, run.Length);
- var markups = run.Markups;
+ // Data is not drawn: no emitter sees it, and text that carries nothing else is written as plain text.
+ var markups = WithoutData(run.Markups);
+ if (markups is null)
+ {
+ EncodeStretch(content, run.Start, run.End, format.Encoding, output);
+ return false;
+ }
var points = CountPoints(markups);
var front = new PooledCharWriter(body.Length + 16);
@@ -499,6 +505,21 @@ private static bool WritePoints(
return written;
}
+ /// The run's layers with its data layers taken out, or null when it carried nothing else.
+ private static MarkupSet? WithoutData(MarkupSet markups)
+ {
+ var data = 0;
+ foreach (var markup in markups)
+ if (markup is IDataMarkup) data++;
+ if (data == 0) return markups;
+ if (data == markups.Count) return null;
+
+ var rest = new List(markups.Count - data);
+ foreach (var markup in markups)
+ if (markup is not IDataMarkup) rest.Add(markup);
+ return MarkupSet.Of(rest);
+ }
+
/// The run's layers with the points taken out, or null when it carried nothing else.
private static MarkupSet? Without(MarkupSet markups, int points)
{
diff --git a/MarkupString/PublicAPI.Unshipped.txt b/MarkupString/PublicAPI.Unshipped.txt
index ea09587..9acac77 100644
--- a/MarkupString/PublicAPI.Unshipped.txt
+++ b/MarkupString/PublicAPI.Unshipped.txt
@@ -1,3 +1,6 @@
#nullable enable
MarkupString.TextEncoding.Markdown = 4 -> MarkupString.TextEncoding
static readonly MarkupString.MarkupFormat.Markdown -> MarkupString.MarkupFormat!
+MarkupString.IDataMarkup
+MarkupString.MarkupText.WithoutVisualMarkup() -> MarkupString.MarkupText!
+MarkupString.MarkupText.WithoutDataMarkup() -> MarkupString.MarkupText!
diff --git a/docs/custom-markup.md b/docs/custom-markup.md
index b18bb58..fb860ba 100644
--- a/docs/custom-markup.md
+++ b/docs/custom-markup.md
@@ -19,6 +19,22 @@ using MarkupString;
public sealed record SpoilerMarkup(string? Reason) : IMarkup;
```
+### Data markup
+
+A layer that says what the text *is*, rather than how it looks, implements `IDataMarkup` instead:
+that a character separates the items of a list, say, and is not part of one.
+
+```csharp
+public sealed record ListSeparator : IDataMarkup;
+```
+
+A data layer has a codec and no emitters. The renderer takes it off a run before any emitter sees
+the run, so the text is written as the layers around it write it, and an emitter registered for a
+data type is never called. `WithoutVisualMarkup()` takes every other layer off and keeps the data
+([Transforming](text-operations.md#transforming)). Everything else treats it as any layer: slicing,
+concatenation and joining carry it, equality ignores it, and a reader without its codec keeps it as
+an `UnknownMarkup`.
+
## 2. The emitters
Two shapes, and the difference matters:
diff --git a/docs/text-operations.md b/docs/text-operations.md
index 7d8fe1b..65fb31d 100644
--- a/docs/text-operations.md
+++ b/docs/text-operations.md
@@ -138,6 +138,21 @@ text.Map(part => part.Trim(TrimType.TrimBoth)); // each run and each plain gap o
returns plain text, because the run positions no longer mean anything. `Map` hands you each styled
run and each unmarked gap as its own `MarkupText` and concatenates what you return.
+```csharp
+text.WithoutVisualMarkup(); // the plain text, with data layers (IDataMarkup) kept
+```
+
+`WithoutVisualMarkup` is what stripping colour means when some layers are data: the result is
+`ToPlainText()`'s text, points dropped with their carriers, and every `IDataMarkup` layer still on
+the characters it covered. Text with no data comes back plain.
+
+```csharp
+text.WithoutDataMarkup(); // the same text and colour, with every data layer taken off
+```
+
+`WithoutDataMarkup` is the other half: it keeps the visual layers and points and drops the data, for
+text from someone who should not be able to say what it is.
+
## Comparing
```csharp