Ask WHO where WHO is known to answer, even when the game states a count

CLAUDE.md

@@ -81,6 +81,13 @@ our host is unreachable and perfectly alive.
   caller that cannot answer keeps asking; only the crawl loop, which has the catalogue, ever sets it.
   `ChineseCount` is what makes those screens answer — it reads the count clause by clause and binds
   a number to 玩家 rather than to the wizards and the people still at the login prompt beside it.
+  **The one exception is an address known to answer `WHO`** (`crawl_target.who_answers_at`, migration
+  0043, `ProbeTarget.WhoAnswers`): there `WHO` is a working command rather than a word at a login
+  prompt, so it is asked and its count outranks a stated one. A stated count is not always the number
+  online — `tapestries.fur.com:2069` answers the plaintext `MSSP-REQUEST` with `PLAYERS = 26842`, its
+  player objects, while its `WHO` counts a few hundred — and before this the report talked the probe
+  out of the `WHO` and the site published the database as the population. `WhoAnswers.Learn` sets it
+  on a counted `WHO` and clears it the first time the login prompt takes `WHO` for a name.
 - **Type `INFO` or `VERSION` at a game whose report already names it and its engine.** Same
   principle as `WHO`, and the same complaint: at `playdecay.com:3003` the old probe had `WHO` taken
   as a character name, was asked for a password, and sent `INFO` as the password — *Wrong password*

migrations/0043_who_answers.sql

@@ -0,0 +1,44 @@
+-- crawl_target.who_answers_at: when this address last showed that WHO is a command it answers.
+--
+-- A game may state its own count, over MSSP or on its connect screen, and the probe then does not
+-- type WHO at it (TelnetProbe.PublishedCountAsync). That restraint is right for a game whose login
+-- prompt reads WHO as a character name. It is wrong for one whose stated count is not the number
+-- online: tapestries.fur.com:2069 answers the plaintext MSSP-REQUEST with PLAYERS = 26842, its player
+-- objects, while its pre-login WHO had counted 300 to 500 every crawl. Where WHO is known to work, WHO
+-- is asked and its count outranks MSSP PLAYERS, as PresenceChoice already ranks them.
+--
+--   NULL   not known to answer WHO. A stated count still spares it the WHO.
+--   set    a WHO typed here came back with a count. Asked every session from then on. Cleared the
+--          first time WHO comes back as the login prompt taking it for a character name.
+--
+-- A measurement of them, like mssp_route: whether a server answers a pre-login WHO is a fact about
+-- the server. Kept off every public surface all the same.
+--
+-- No BEGIN/COMMIT: MigrationRunner opens its own transaction per script and writes the ledger entry
+-- inside it.
+
+ALTER TABLE crawl_target
+    ADD COLUMN who_answers_at timestamptz;
+
+COMMENT ON COLUMN crawl_target.who_answers_at IS
+    'When WHO typed at this address last came back with a count. Set: WHO is asked every session '
+    'even where the game states its own count. NULL: not known to answer WHO.';
+
+-- A `who` presence row exists only where a WHO came back counted and was published, so it is
+-- exactly the evidence this column records. presence_sample is keyed by game, not address, so only a
+-- game with one address can be credited without guessing which listener answered; a game on several
+-- addresses learns it per address on the next WHO that counts. Dated by the newest such row, which is
+-- when it was last seen to be true.
+UPDATE crawl_target t
+   SET who_answers_at = w.at
+  FROM (SELECT p.game_id, max(p.at) AS at
+          FROM presence_sample p
+         WHERE p.source = 'who'
+           AND p.count IS NOT NULL
+         GROUP BY p.game_id) w
+ WHERE t.game_id = w.game_id
+   AND NOT EXISTS (
+        SELECT 1
+          FROM crawl_target other
+         WHERE other.game_id = t.game_id
+           AND other.id <> t.id);

src/MUI.Crawl/Telnet/Probing.cs

@@ -102,6 +102,25 @@ public sealed record ProbeTarget(string Host, int Port)
     /// </remarks>
     public PlaintextMsspAsk PlaintextMssp { get; init; } = PlaintextMsspAsk.Never;
 
+    /// <summary>
+    /// Whether this address is known to answer a pre-login <c>WHO</c> with a count.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// True means <c>WHO</c> is asked even where the game has already stated a count, and its count
+    /// is the one published. A stated count is the game's claim; <c>WHO</c> is the live answer, and
+    /// the two are not always the same number: <c>tapestries.fur.com:2069</c> answers the plaintext
+    /// <c>MSSP-REQUEST</c> with <c>PLAYERS = 26842</c>, its player objects, while its <c>WHO</c>
+    /// counts a few hundred.
+    /// </para>
+    /// <para>
+    /// False by default, so a caller that cannot say keeps the restraint
+    /// <c>TelnetProbe.PublishedCountAsync</c> describes. Only the crawl loop answers it, from
+    /// <c>crawl_target.who_answers_at</c> (migration 0043).
+    /// </para>
+    /// </remarks>
+    public bool WhoAnswers { get; init; }
+
     public override string ToString() => Host.Contains(':') ? $"[{Host}]:{Port}" : $"{Host}:{Port}";
 }
 

src/MUI.Crawl/Telnet/TelnetProbe.cs

@@ -956,7 +956,14 @@ async Task<int> AskAsync(string command, int baseline, TimeSpan grace, Action? s
             // answer we have been given is not a measurement, it is noise on somebody's console.
             // Decided here rather than earlier because here is the latest moment before the send,
             // so an MSSP report still in flight through the flush above is counted.
-            published = await PublishedCountAsync(lines, seen, cursors, target, cancellationToken);
+            //
+            // Unless this address is known to answer WHO. Then WHO is a working command rather than a
+            // word at a login prompt, and its count is the live one: a stated count can be something
+            // else entirely (see ProbeTarget.WhoAnswers). Asking means not-asking-implies-publishing
+            // has nothing to hold, and PresenceChoice still falls back to MSSP if this WHO fails.
+            published = target.WhoAnswers
+                ? null
+                : await PublishedCountAsync(lines, seen, cursors, target, cancellationToken);
 
             if (Live(client) && !whoAlreadyAnswered && published is null)
             {

src/MUI.Crawler/Crawl/CrawlCycle.cs

@@ -263,6 +263,10 @@ private async Task<Attempt> AttemptAsync(CrawlTarget target, CancellationToken c
                 // Whether to type the plaintext MSSP-REQUEST, from what this address did with it
                 // before. See MsspRoutes for both halves of that memory.
                 PlaintextMssp = MsspRoutes.Ask(target.MsspRoute),
+
+                // Whether WHO is a working command here, so a stated count does not stand in for it.
+                // See WhoAnswers for how the memory changes.
+                WhoAnswers = target.WhoAnswersAt is not null,
             },
             budget.Token);
 
@@ -518,6 +522,13 @@ await targets.RecordAttemptAsync(
                     route is MsspRoute.Plaintext ? "answered" : "did not answer");
             }
         }
+
+        // And whether WHO works here, on the same terms.
+        if (WhoAnswers.Learn(target.WhoAnswersAt, result, now) is var whoAnswers
+            && whoAnswers != target.WhoAnswersAt)
+        {
+            await targets.RecordWhoAnswersAsync(target.Id, whoAnswers, cancellationToken);
+        }
     }
 
     /// <summary>

src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs

@@ -42,7 +42,7 @@ public sealed class NpgsqlCrawlTargetRepository(NpgsqlDataSource source) : ICraw
         crawl_delay AS CrawlDelay, first_seen_at AS FirstSeenAt, last_probed_at AS LastProbedAt,
         discovered_from_game_id AS DiscoveredFromGameId, depth AS Depth,
         is_operator_seed AS IsOperatorSeed, submitted_at AS SubmittedAt,
-        discovered_via AS DiscoveredVia, mssp_route AS MsspRoute,
+        discovered_via AS DiscoveredVia, mssp_route AS MsspRoute, who_answers_at AS WhoAnswersAt,
         (SELECT f.value
            FROM game_field f
           WHERE f.game_id = crawl_target.game_id
@@ -204,6 +204,16 @@ UPDATE crawl_target
             cancellationToken: ct));
     }
 
+    public async Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct)
+    {
+        await using var connection = await source.OpenConnectionAsync(ct);
+
+        await connection.ExecuteAsync(new CommandDefinition(
+            "UPDATE crawl_target SET who_answers_at = @at WHERE id = @id",
+            new { id, at = at?.ToUniversalTime() },
+            cancellationToken: ct));
+    }
+
     /// <summary>
     /// Attaches the game this address turned out to be.
     /// </summary>
@@ -262,6 +272,8 @@ private sealed class Row
         /// <remarks>Text, not the enum, for the reason <see cref="DiscoveredVia"/> is.</remarks>
         public string? MsspRoute { get; init; }
 
+        public DateTimeOffset? WhoAnswersAt { get; init; }
+
         /// <remarks>
         /// Nullable because the subquery has no row to read for a target not yet bound to a game,
         /// and null there means the same as true: an address that has proved nothing yet is asked
@@ -291,6 +303,7 @@ private sealed class Row
             MsspCharset = MsspCharset,
             AwaitingCorroboration = AwaitingCorroboration ?? true,
             MsspRoute = MsspRouteSpelling.From(MsspRoute),
+            WhoAnswersAt = WhoAnswersAt,
         };
     }
 }

src/MUI.Discovery/Scheduling/CrawlTarget.cs

@@ -140,6 +140,12 @@ public sealed record CrawlTarget
     /// asks for and how it changes.
     /// </remarks>
     public MsspRoute? MsspRoute { get; init; }
+
+    /// <summary>
+    /// When a <c>WHO</c> typed at this address last came back with a count, or null when it is not
+    /// known to answer one (migration 0043). <see cref="WhoAnswers"/> owns how it changes.
+    /// </summary>
+    public DateTimeOffset? WhoAnswersAt { get; init; }
 }
 
 /// <summary>
@@ -193,5 +199,12 @@ Task RecordAttemptAsync(
     /// </remarks>
     Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct);
 
+    /// <summary>
+    /// Records that this address answers <c>WHO</c> with a count, as of <paramref name="at"/>, or
+    /// with null that it no longer does. Written only on a change, like
+    /// <see cref="RecordMsspRouteAsync"/>.
+    /// </summary>
+    Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct);
+
     Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct);
 }

src/MUI.Discovery/Scheduling/WhoAnswers.cs

@@ -0,0 +1,50 @@
+using MUI.Crawl;
+
+namespace MUI.Discovery;
+
+/// <summary>
+/// What a probe teaches about whether an address answers a pre-login <c>WHO</c>
+/// (<c>crawl_target.who_answers_at</c>, migration 0043).
+/// </summary>
+/// <remarks>
+/// A pure function of a target's memory and a <see cref="ProbeResult"/>, as <see cref="MsspRoutes"/>
+/// is, so it is testable without a socket or a database.
+/// </remarks>
+public static class WhoAnswers
+{
+    /// <summary>
+    /// The new value of <see cref="CrawlTarget.WhoAnswersAt"/>, or <paramref name="current"/> when
+    /// this probe taught nothing.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// A counted <c>WHO</c> sets it, dated by the probe, and only from unknown: re-dating it on every
+    /// crawl would be a write per probe to say what the row already says.
+    /// </para>
+    /// <para>
+    /// The login prompt taking <c>WHO</c> for a character name clears it, because that is the game
+    /// saying it has no pre-login <c>WHO</c> at this address any more, and typing it again every
+    /// crawl is exactly what the restraint in <c>TelnetProbe.PublishedCountAsync</c> exists to stop.
+    /// An unreadable answer does not: that is our parser meeting a dialect, not the game changing.
+    /// </para>
+    /// <para>
+    /// A failed dial teaches nothing, and neither does a session that never typed <c>WHO</c>.
+    /// </para>
+    /// </remarks>
+    public static DateTimeOffset? Learn(DateTimeOffset? current, ProbeResult result, DateTimeOffset now)
+    {
+        ArgumentNullException.ThrowIfNull(result);
+
+        if (result.Outcome is not ProbeOutcome.Answered)
+        {
+            return current;
+        }
+
+        if (result.Who.HasCount)
+        {
+            return current ?? now;
+        }
+
+        return result.Who.Confidence is WhoConfidence.LoginPrompt ? null : current;
+    }
+}

tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs

@@ -151,6 +151,29 @@ public async Task AKnownAnswererIsAskedBeforeWhoAndSparedTheWho()
         await Assert.That(result.Banner).Contains("Welcome to Old Realms");
     }
 
+    /// <summary>
+    /// A stated count does not stand in for <c>WHO</c> at an address known to answer it.
+    /// </summary>
+    /// <remarks>
+    /// <c>tapestries.fur.com:2069</c> answers <c>MSSP-REQUEST</c> with <c>PLAYERS = 26842</c>, its
+    /// player objects, while its <c>WHO</c> counts a few hundred. The report spared it the
+    /// <c>WHO</c>, and the site published the database as the population.
+    /// </remarks>
+    [Test]
+    public async Task AKnownAnswererKnownToAnswerWhoIsStillAskedWho()
+    {
+        await using var game = new OldRealms { AnswersRequest = true };
+
+        var result = await new TelnetProbe(Fast()).ProbeAsync(
+            game.Target with { PlaintextMssp = PlaintextMsspAsk.Ask, WhoAnswers = true });
+
+        await Assert.That(MsspReport.Last(result.Mssp, "PLAYERS")).IsEqualTo("4");
+        await Assert.That(game.Sessions).Count().IsEqualTo(1);
+        await Assert.That(game.Sessions[0].First()).IsEqualTo(Request);
+        await Assert.That(game.Sessions[0]).Contains("WHO");
+        await Assert.That(result.Who.Count).IsEqualTo(3);
+    }
+
     [Test]
     public async Task AKnownAnswererThatFallsSilentIsRecordedAsSuch()
     {

tests/MUI.Crawler.Tests/Crawl/WhoAnswersPostgresTests.cs

@@ -0,0 +1,68 @@
+using Dapper;
+
+using MUI.Crawl;
+using MUI.Crawler.Tests.Support;
+
+namespace MUI.Crawler.Tests;
+
+/// <summary>
+/// What the crawl loop remembers about whether an address answers <c>WHO</c>
+/// (<c>crawl_target.who_answers_at</c>), and that the next dial is told.
+/// </summary>
+public class WhoAnswersPostgresTests
+{
+    private const string Host = "tapestries.example.org";
+    private const int Port = 2069;
+
+    [Test]
+    public async Task AnAddressWhoseWhoCountedIsAskedWhoFromThenOn()
+    {
+        await using var database = await PostgresFixture.MigratedAsync();
+        var source = database.DataSource;
+
+        await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port));
+
+        var first = new ScriptedProbe(target => Probes.Answered(
+            host: target.Host, port: target.Port, who: new WhoReading(WhoConfidence.Count, 412)));
+        await CrawlCycles.Build(source, first, TimeProvider.System).RunAsync();
+
+        await Assert.That(first.Asked.Single().WhoAnswers).IsFalse();
+
+        await using var connection = await source.OpenConnectionAsync();
+
+        await Assert.That(await KnownAsync(connection)).IsTrue();
+
+        await DueAgainAsync(connection);
+
+        var next = new ScriptedProbe(target => Probes.Answered(
+            host: target.Host, port: target.Port, who: new WhoReading(WhoConfidence.Count, 405)));
+        await CrawlCycles.Build(source, next, TimeProvider.System).RunAsync();
+
+        await Assert.That(next.Asked.Single().WhoAnswers).IsTrue();
+    }
+
+    [Test]
+    public async Task AnAddressWhoseLoginPromptAteWhoIsForgotten()
+    {
+        await using var database = await PostgresFixture.MigratedAsync();
+        var source = database.DataSource;
+
+        await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port));
+
+        await using var connection = await source.OpenConnectionAsync();
+        await connection.ExecuteAsync("UPDATE crawl_target SET who_answers_at = now() - interval '1 day'");
+
+        var probe = new ScriptedProbe(target => Probes.Answered(
+            host: target.Host, port: target.Port, who: WhoReading.LoginPrompt));
+        await CrawlCycles.Build(source, probe, TimeProvider.System).RunAsync();
+
+        await Assert.That(probe.Asked.Single().WhoAnswers).IsTrue();
+        await Assert.That(await KnownAsync(connection)).IsFalse();
+    }
+
+    private static Task<bool> KnownAsync(Npgsql.NpgsqlConnection connection) =>
+        connection.ExecuteScalarAsync<bool>("SELECT who_answers_at IS NOT NULL FROM crawl_target");
+
+    private static Task DueAgainAsync(Npgsql.NpgsqlConnection connection) =>
+        connection.ExecuteAsync("UPDATE crawl_target SET next_probe_at = now() - interval '1 hour'");
+}

tests/MUI.Crawler.Tests/Support/FakeStores.cs

@@ -66,6 +66,8 @@ public Task RecordAttemptAsync(
     public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) =>
         Task.CompletedTask;
 
+    public Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct) => Task.CompletedTask;
+
     public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) => Task.CompletedTask;
 }