Data markup: layers that say what text is, kept by WithoutVisualMarkup()

CHANGELOG.md

@@ -6,7 +6,20 @@ and `MarkupString.Pueblo`. The packages share one version and are released toget
 The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project
 follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## 2.20.0 — Unreleased
+## 2.21.0 — Unreleased
+
+### Added
+
+- **Data markup.** `IDataMarkup` marks a layer that says what the text is rather than how it looks,
+  such as a list separator. No format draws it: the renderer takes it off each run before any emitter
+  sees the run, so the text is written as the layers around it write it. It is stored, carried and
+  compared like any other layer. See [Data markup](docs/custom-markup.md#data-markup).
+- **`MarkupText.WithoutVisualMarkup()`**: the plain text with every data layer kept, for stripping
+  colour from text that also carries data.
+- **`MarkupText.WithoutDataMarkup()`**: the text with every data layer taken off and its visual
+  markup and points kept, for text whose data must not be trusted.
+
+## 2.20.0
 
 ### Added
 

MarkupString.Tests/DataMarkupTests.cs

@@ -0,0 +1,168 @@
+using System.Text.Json;
+using MarkupString.Ansi;
+using MarkupString.Html;
+namespace MarkupString.Tests;
+
+/// <summary>
+/// Data markup says what text is, not how it looks: no format draws it, taking the visual markup off keeps
+/// it, and it is carried and stored like any other layer.
+/// </summary>
+public class DataMarkupTests
+{
+	private sealed record Separator : IDataMarkup;
+
+	private sealed class SeparatorCodec : IMarkupCodec
+	{
+		public string Kind => "test-separator";
+		public Type MarkupType => typeof(Separator);
+		public void Write(Utf8JsonWriter writer, IMarkup markup) { }
+		public IMarkup Read(JsonElement element) => new Separator();
+	}
+
+	/// <summary>Draws a separator as <c>{|}</c>, to show that the renderer never calls an emitter for data.</summary>
+	private sealed class SeparatorEmitter : IMarkupEmitter
+	{
+		public Type MarkupType => typeof(Separator);
+		public MarkupFormat Format => MarkupFormat.Ansi;
+		public void Emit(IMarkup markup, ReadOnlySpan<char> body, in EmitContext context, System.Buffers.IBufferWriter<char> output)
+		{
+			output.Write("{");
+			output.Write(body);
+			output.Write("}");
+		}
+	}
+
+	private static readonly MarkupRegistry Registry = MarkupRegistry.Empty.WithAnsi().WithHtml().With(new SeparatorCodec());
+
+	private static readonly AnsiMarkup Red = AnsiMarkup.Create(foreground: new AnsiColor.Standard(1, false));
+
+	private static MarkupText Marked(string text) => MarkupText.Wrap(new Separator(), text);
+
+	private static bool IsData(MarkupText text, int index) =>
+		text.Runs.Any(run => index >= run.Start && index < run.End && run.Markups.Any(markup => markup is IDataMarkup));
+
+	[Test]
+	public async Task DataIsWrittenAsItsText()
+	{
+		var text = MarkupText.Concat([MarkupText.Plain("a"), Marked("|"), MarkupText.Plain("b")]);
+
+		await Assert.That(text.Render(MarkupFormat.Ansi, Registry)).IsEqualTo("a|b");
+		await Assert.That(text.Render(MarkupFormat.Html, Registry)).IsEqualTo("a|b");
+		await Assert.That(text.Render(MarkupFormat.Plain, Registry)).IsEqualTo("a|b");
+	}
+
+	[Test]
+	public async Task DataUnderColourIsWrittenInTheColour()
+	{
+		var coloured = MarkupText.Concat([MarkupText.Plain("a"), MarkupText.Wrap(Red, Marked("|")), MarkupText.Plain("b")]);
+		var colourOnly = MarkupText.Concat([MarkupText.Plain("a"), MarkupText.Wrap(Red, "|"), MarkupText.Plain("b")]);
+
+		await Assert.That(coloured.Render(MarkupFormat.Ansi, Registry)).IsEqualTo(colourOnly.Render(MarkupFormat.Ansi, Registry));
+		await Assert.That(coloured.Render(MarkupFormat.Html, Registry)).IsEqualTo(colourOnly.Render(MarkupFormat.Html, Registry));
+	}
+
+	[Test]
+	public async Task AnEmitterForDataIsNeverCalled()
+	{
+		var registry = Registry.With(new SeparatorEmitter());
+
+		await Assert.That(MarkupText.Concat([MarkupText.Plain("a"), Marked("|")]).Render(MarkupFormat.Ansi, registry)).IsEqualTo("a|");
+	}
+
+	[Test]
+	public async Task WithoutVisualMarkup_KeepsTheDataAndDropsTheColour()
+	{
+		var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Wrap(Red, Marked("|")), MarkupText.Plain("b"), Marked("|")]);
+
+		var stripped = text.WithoutVisualMarkup();
+
+		await Assert.That(stripped.Text).IsEqualTo("a|b|");
+		await Assert.That(stripped.Runs.Length).IsEqualTo(2);
+		await Assert.That(stripped.Runs.All(run => run.Markups.Count == 1 && run.Markups[0] is Separator)).IsTrue();
+		await Assert.That(IsData(stripped, 1)).IsTrue();
+		await Assert.That(IsData(stripped, 3)).IsTrue();
+		await Assert.That(IsData(stripped, 0)).IsFalse();
+	}
+
+	[Test]
+	public async Task WithoutVisualMarkup_JoinsDataThatColourHadSplit()
+	{
+		var text = MarkupText.Concat([MarkupText.Wrap(Red, Marked("|")), Marked("|")]);
+
+		var stripped = text.WithoutVisualMarkup();
+
+		await Assert.That(stripped.Runs.Length).IsEqualTo(1);
+		await Assert.That(stripped.Runs[0].Length).IsEqualTo(2);
+	}
+
+	[Test]
+	public async Task WithoutVisualMarkup_WithNoData_IsThePlainText()
+	{
+		var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Bell(), MarkupText.Plain("b")]);
+
+		var stripped = text.WithoutVisualMarkup();
+
+		await Assert.That(stripped.Text).IsEqualTo("ab");
+		await Assert.That(stripped.Runs.IsDefaultOrEmpty).IsTrue();
+	}
+
+	[Test]
+	public async Task WithoutVisualMarkup_DropsPointsWithTheirCarriers()
+	{
+		var text = MarkupText.Concat([Marked("|"), MarkupText.Bell(), MarkupText.Plain("b")]);
+
+		var stripped = text.WithoutVisualMarkup();
+
+		await Assert.That(stripped.Text).IsEqualTo("|b");
+		await Assert.That(IsData(stripped, 0)).IsTrue();
+	}
+
+	[Test]
+	public async Task WithoutDataMarkup_KeepsTheColourAndDropsTheData()
+	{
+		var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Wrap(Red, Marked("|")), MarkupText.Plain("b"), Marked("|")]);
+
+		var unmarked = text.WithoutDataMarkup();
+
+		await Assert.That(unmarked.Text).IsEqualTo("a|b|");
+		await Assert.That(IsData(unmarked, 1)).IsFalse();
+		await Assert.That(IsData(unmarked, 3)).IsFalse();
+		await Assert.That(unmarked.Render(MarkupFormat.Ansi, Registry))
+			.IsEqualTo(MarkupText.Concat([MarkupText.Wrap(Red, "a|"), MarkupText.Plain("b|")]).Render(MarkupFormat.Ansi, Registry));
+	}
+
+	[Test]
+	public async Task WithoutDataMarkup_KeepsPoints()
+	{
+		var text = MarkupText.Concat([Marked("|"), MarkupText.Bell(), MarkupText.Plain("b")]);
+
+		var unmarked = text.WithoutDataMarkup();
+
+		await Assert.That(unmarked.Text).IsEqualTo(text.Text);
+		await Assert.That(unmarked.ToPlainText()).IsEqualTo("|b");
+		await Assert.That(IsData(unmarked, 0)).IsFalse();
+	}
+
+	[Test]
+	public async Task WithoutDataMarkup_WithNoData_IsTheSameText()
+	{
+		var text = MarkupText.Concat([MarkupText.Wrap(Red, "a"), MarkupText.Plain("b")]);
+
+		await Assert.That(ReferenceEquals(text.WithoutDataMarkup(), text)).IsTrue();
+	}
+
+	[Test]
+	public async Task DataIsIgnoredByEquality()
+		=> await Assert.That(MarkupText.Concat([MarkupText.Plain("a"), Marked("|")])).IsEqualTo(MarkupText.Plain("a|"));
+
+	[Test]
+	public async Task DataIsStoredAndReadBack()
+	{
+		var text = MarkupText.Concat([MarkupText.Plain("a"), Marked("|"), MarkupText.Plain("b")]);
+
+		var back = MarkupTextSerializer.Deserialize(MarkupTextSerializer.Serialize(text, Registry), Registry);
+
+		await Assert.That(IsData(back, 1)).IsTrue();
+		await Assert.That(IsData(back, 0)).IsFalse();
+	}
+}

MarkupString/IDataMarkup.cs

@@ -0,0 +1,16 @@
+namespace MarkupString;
+
+/// <summary>
+/// A markup that says what the text it covers is, rather than how it looks: that a character is a list
+/// separator, say, and not part of an item.
+/// </summary>
+/// <remarks>
+/// <para>No format writes anything for a data layer. The renderer leaves it out before any emitter sees the
+/// run, so the text it covers is written as the layers around it would write it, and an emitter registered
+/// for a data type is never called.</para>
+/// <para>Data stays where visual markup goes: <see cref="MarkupText.WithoutVisualMarkup"/> takes every other
+/// layer off and keeps these. Like every layer, it is ignored by <see cref="MarkupText.ToPlainText"/> and
+/// equality, carried through slicing, concatenation and joining, and stored through its codec
+/// (<see cref="IMarkupCodec"/>); a reader without the codec keeps it as an <see cref="UnknownMarkup"/>.</para>
+/// </remarks>
+public interface IDataMarkup : IMarkup;

MarkupString/MarkupText.cs

@@ -154,6 +154,78 @@ public static MarkupText Join(Func<int, MarkupText> separator, IEnumerable<Marku
 	/// </summary>
 	public string ToPlainText() => _plainText ??= WithoutPoints();
 
+	/// <summary>
+	/// The text with every visual layer taken off and its data layers (<see cref="IDataMarkup"/>) kept: what
+	/// <see cref="ToPlainText"/> gives, plus the data. Points go with their carriers, as they do there.
+	/// </summary>
+	public MarkupText WithoutVisualMarkup()
+	{
+		if (Runs.IsDefaultOrEmpty) return this;
+
+		var hasData = false;
+		foreach (var run in Runs)
+		{
+			foreach (var markup in run.Markups)
+			{
+				if (markup is not IDataMarkup) continue;
+				hasData = true;
+				break;
+			}
+			if (hasData) break;
+		}
+		if (!hasData) return Plain(ToPlainText());
+
+		var parts = new List<MarkupText>(Runs.Length * 2 + 1);
+		var position = 0;
+		foreach (var run in Runs)
+		{
+			if (run.Start > position) parts.Add(Plain(Text[position..run.Start]));
+			position = run.End;
+			if (IsPoint(run)) continue;
+
+			var data = new List<IMarkup>();
+			foreach (var markup in run.Markups)
+				if (markup is IDataMarkup) data.Add(markup);
+			var body = Text.Substring(run.Start, run.Length);
+			parts.Add(data.Count == 0 ? Plain(body) : Wrap(MarkupSet.Of(data), body));
+		}
+		if (position < Text.Length) parts.Add(Plain(Text[position..]));
+		return Concat(parts);
+	}
+
+	/// <summary>
+	/// The text with every data layer (<see cref="IDataMarkup"/>) taken off and everything else kept: the
+	/// counterpart of <see cref="WithoutVisualMarkup"/>, for text whose data must not be trusted.
+	/// </summary>
+	public MarkupText WithoutDataMarkup()
+	{
+		if (Runs.IsDefaultOrEmpty) return this;
+
+		var hasData = false;
+		foreach (var run in Runs)
+		{
+			foreach (var markup in run.Markups)
+			{
+				if (markup is not IDataMarkup) continue;
+				hasData = true;
+				break;
+			}
+			if (hasData) break;
+		}
+		if (!hasData) return this;
+
+		var builder = ImmutableArray.CreateBuilder<Run>(Runs.Length);
+		foreach (var run in Runs)
+		{
+			var kept = new List<IMarkup>(run.Markups.Count);
+			foreach (var markup in run.Markups)
+				if (markup is not IDataMarkup) kept.Add(markup);
+			if (kept.Count == run.Markups.Count) builder.Add(run);
+			else if (kept.Count > 0) builder.Add(run with { Markups = MarkupSet.Of(kept) });
+		}
+		return new MarkupText(Text, builder.ToImmutable());
+	}
+
 	/// <summary>The same as <see cref="ToPlainText"/>.</summary>
 	public override string ToString() => ToPlainText();
 	public bool Equals(MarkupText? other) => other is not null && string.Equals(ToPlainText(), other.ToPlainText(), StringComparison.Ordinal);

MarkupString/MarkupTextRenderer.cs

@@ -379,7 +379,13 @@ private static bool RenderRun(
 		IBufferWriter<char> output)
 	{
 		var body = content.Slice(run.Start, run.Length);
-		var markups = run.Markups;
+		// Data is not drawn: no emitter sees it, and text that carries nothing else is written as plain text.
+		var markups = WithoutData(run.Markups);
+		if (markups is null)
+		{
+			EncodeStretch(content, run.Start, run.End, format.Encoding, output);
+			return false;
+		}
 		var points = CountPoints(markups);
 
 		var front = new PooledCharWriter(body.Length + 16);
@@ -499,6 +505,21 @@ private static bool WritePoints(
 		return written;
 	}
 
+	/// <summary>The run's layers with its data layers taken out, or null when it carried nothing else.</summary>
+	private static MarkupSet? WithoutData(MarkupSet markups)
+	{
+		var data = 0;
+		foreach (var markup in markups)
+			if (markup is IDataMarkup) data++;
+		if (data == 0) return markups;
+		if (data == markups.Count) return null;
+
+		var rest = new List<IMarkup>(markups.Count - data);
+		foreach (var markup in markups)
+			if (markup is not IDataMarkup) rest.Add(markup);
+		return MarkupSet.Of(rest);
+	}
+
 	/// <summary>The run's layers with the points taken out, or null when it carried nothing else.</summary>
 	private static MarkupSet? Without(MarkupSet markups, int points)
 	{

MarkupString/PublicAPI.Unshipped.txt

@@ -1,3 +1,6 @@
 #nullable enable
 MarkupString.TextEncoding.Markdown = 4 -> MarkupString.TextEncoding
 static readonly MarkupString.MarkupFormat.Markdown -> MarkupString.MarkupFormat!
+MarkupString.IDataMarkup
+MarkupString.MarkupText.WithoutVisualMarkup() -> MarkupString.MarkupText!
+MarkupString.MarkupText.WithoutDataMarkup() -> MarkupString.MarkupText!

docs/custom-markup.md

@@ -19,6 +19,22 @@ using MarkupString;
 public sealed record SpoilerMarkup(string? Reason) : IMarkup;
 ```
 
+### Data markup
+
+A layer that says what the text *is*, rather than how it looks, implements `IDataMarkup` instead:
+that a character separates the items of a list, say, and is not part of one.
+
+```csharp
+public sealed record ListSeparator : IDataMarkup;
+```
+
+A data layer has a codec and no emitters. The renderer takes it off a run before any emitter sees
+the run, so the text is written as the layers around it write it, and an emitter registered for a
+data type is never called. `WithoutVisualMarkup()` takes every other layer off and keeps the data
+([Transforming](text-operations.md#transforming)). Everything else treats it as any layer: slicing,
+concatenation and joining carry it, equality ignores it, and a reader without its codec keeps it as
+an `UnknownMarkup`.
+
 ## 2. The emitters
 
 Two shapes, and the difference matters:

docs/text-operations.md

@@ -138,6 +138,21 @@ text.Map(part => part.Trim(TrimType.TrimBoth)); // each run and each plain gap o
 returns plain text, because the run positions no longer mean anything. `Map` hands you each styled
 run and each unmarked gap as its own `MarkupText` and concatenates what you return.
 
+```csharp
+text.WithoutVisualMarkup();   // the plain text, with data layers (IDataMarkup) kept
+```
+
+`WithoutVisualMarkup` is what stripping colour means when some layers are data: the result is
+`ToPlainText()`'s text, points dropped with their carriers, and every `IDataMarkup` layer still on
+the characters it covered. Text with no data comes back plain.
+
+```csharp
+text.WithoutDataMarkup();     // the same text and colour, with every data layer taken off
+```
+
+`WithoutDataMarkup` is the other half: it keeps the visual layers and points and drops the data, for
+text from someone who should not be able to say what it is.
+
 ## Comparing
 
 ```csharp