diff --git a/CLAUDE.md b/CLAUDE.md index 250361f..671dda0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,6 +81,13 @@ our host is unreachable and perfectly alive. caller that cannot answer keeps asking; only the crawl loop, which has the catalogue, ever sets it. `ChineseCount` is what makes those screens answer — it reads the count clause by clause and binds a number to 玩家 rather than to the wizards and the people still at the login prompt beside it. + **The one exception is an address known to answer `WHO`** (`crawl_target.who_answers_at`, migration + 0043, `ProbeTarget.WhoAnswers`): there `WHO` is a working command rather than a word at a login + prompt, so it is asked and its count outranks a stated one. A stated count is not always the number + online — `tapestries.fur.com:2069` answers the plaintext `MSSP-REQUEST` with `PLAYERS = 26842`, its + player objects, while its `WHO` counts a few hundred — and before this the report talked the probe + out of the `WHO` and the site published the database as the population. `WhoAnswers.Learn` sets it + on a counted `WHO` and clears it the first time the login prompt takes `WHO` for a name. - **Type `INFO` or `VERSION` at a game whose report already names it and its engine.** Same principle as `WHO`, and the same complaint: at `playdecay.com:3003` the old probe had `WHO` taken as a character name, was asked for a password, and sent `INFO` as the password — *Wrong password* diff --git a/migrations/0043_who_answers.sql b/migrations/0043_who_answers.sql new file mode 100644 index 0000000..e8930dd --- /dev/null +++ b/migrations/0043_who_answers.sql @@ -0,0 +1,44 @@ +-- crawl_target.who_answers_at: when this address last showed that WHO is a command it answers. +-- +-- A game may state its own count, over MSSP or on its connect screen, and the probe then does not +-- type WHO at it (TelnetProbe.PublishedCountAsync). That restraint is right for a game whose login +-- prompt reads WHO as a character name. It is wrong for one whose stated count is not the number +-- online: tapestries.fur.com:2069 answers the plaintext MSSP-REQUEST with PLAYERS = 26842, its player +-- objects, while its pre-login WHO had counted 300 to 500 every crawl. Where WHO is known to work, WHO +-- is asked and its count outranks MSSP PLAYERS, as PresenceChoice already ranks them. +-- +-- NULL not known to answer WHO. A stated count still spares it the WHO. +-- set a WHO typed here came back with a count. Asked every session from then on. Cleared the +-- first time WHO comes back as the login prompt taking it for a character name. +-- +-- A measurement of them, like mssp_route: whether a server answers a pre-login WHO is a fact about +-- the server. Kept off every public surface all the same. +-- +-- No BEGIN/COMMIT: MigrationRunner opens its own transaction per script and writes the ledger entry +-- inside it. + +ALTER TABLE crawl_target + ADD COLUMN who_answers_at timestamptz; + +COMMENT ON COLUMN crawl_target.who_answers_at IS + 'When WHO typed at this address last came back with a count. Set: WHO is asked every session ' + 'even where the game states its own count. NULL: not known to answer WHO.'; + +-- A `who` presence row exists only where a WHO came back counted and was published, so it is +-- exactly the evidence this column records. presence_sample is keyed by game, not address, so only a +-- game with one address can be credited without guessing which listener answered; a game on several +-- addresses learns it per address on the next WHO that counts. Dated by the newest such row, which is +-- when it was last seen to be true. +UPDATE crawl_target t + SET who_answers_at = w.at + FROM (SELECT p.game_id, max(p.at) AS at + FROM presence_sample p + WHERE p.source = 'who' + AND p.count IS NOT NULL + GROUP BY p.game_id) w + WHERE t.game_id = w.game_id + AND NOT EXISTS ( + SELECT 1 + FROM crawl_target other + WHERE other.game_id = t.game_id + AND other.id <> t.id); diff --git a/src/MUI.Crawl/Telnet/Probing.cs b/src/MUI.Crawl/Telnet/Probing.cs index 4c6f857..52f9ec2 100644 --- a/src/MUI.Crawl/Telnet/Probing.cs +++ b/src/MUI.Crawl/Telnet/Probing.cs @@ -102,6 +102,25 @@ public sealed record ProbeTarget(string Host, int Port) /// public PlaintextMsspAsk PlaintextMssp { get; init; } = PlaintextMsspAsk.Never; + /// + /// Whether this address is known to answer a pre-login WHO with a count. + /// + /// + /// + /// True means WHO is asked even where the game has already stated a count, and its count + /// is the one published. A stated count is the game's claim; WHO is the live answer, and + /// the two are not always the same number: tapestries.fur.com:2069 answers the plaintext + /// MSSP-REQUEST with PLAYERS = 26842, its player objects, while its WHO + /// counts a few hundred. + /// + /// + /// False by default, so a caller that cannot say keeps the restraint + /// TelnetProbe.PublishedCountAsync describes. Only the crawl loop answers it, from + /// crawl_target.who_answers_at (migration 0043). + /// + /// + public bool WhoAnswers { get; init; } + public override string ToString() => Host.Contains(':') ? $"[{Host}]:{Port}" : $"{Host}:{Port}"; } diff --git a/src/MUI.Crawl/Telnet/TelnetProbe.cs b/src/MUI.Crawl/Telnet/TelnetProbe.cs index 7a9601f..f94356a 100644 --- a/src/MUI.Crawl/Telnet/TelnetProbe.cs +++ b/src/MUI.Crawl/Telnet/TelnetProbe.cs @@ -956,7 +956,14 @@ async Task AskAsync(string command, int baseline, TimeSpan grace, Action? s // answer we have been given is not a measurement, it is noise on somebody's console. // Decided here rather than earlier because here is the latest moment before the send, // so an MSSP report still in flight through the flush above is counted. - published = await PublishedCountAsync(lines, seen, cursors, target, cancellationToken); + // + // Unless this address is known to answer WHO. Then WHO is a working command rather than a + // word at a login prompt, and its count is the live one: a stated count can be something + // else entirely (see ProbeTarget.WhoAnswers). Asking means not-asking-implies-publishing + // has nothing to hold, and PresenceChoice still falls back to MSSP if this WHO fails. + published = target.WhoAnswers + ? null + : await PublishedCountAsync(lines, seen, cursors, target, cancellationToken); if (Live(client) && !whoAlreadyAnswered && published is null) { diff --git a/src/MUI.Crawler/Crawl/CrawlCycle.cs b/src/MUI.Crawler/Crawl/CrawlCycle.cs index 2498ff5..8da57bc 100644 --- a/src/MUI.Crawler/Crawl/CrawlCycle.cs +++ b/src/MUI.Crawler/Crawl/CrawlCycle.cs @@ -263,6 +263,10 @@ private async Task AttemptAsync(CrawlTarget target, CancellationToken c // Whether to type the plaintext MSSP-REQUEST, from what this address did with it // before. See MsspRoutes for both halves of that memory. PlaintextMssp = MsspRoutes.Ask(target.MsspRoute), + + // Whether WHO is a working command here, so a stated count does not stand in for it. + // See WhoAnswers for how the memory changes. + WhoAnswers = target.WhoAnswersAt is not null, }, budget.Token); @@ -518,6 +522,13 @@ await targets.RecordAttemptAsync( route is MsspRoute.Plaintext ? "answered" : "did not answer"); } } + + // And whether WHO works here, on the same terms. + if (WhoAnswers.Learn(target.WhoAnswersAt, result, now) is var whoAnswers + && whoAnswers != target.WhoAnswersAt) + { + await targets.RecordWhoAnswersAsync(target.Id, whoAnswers, cancellationToken); + } } /// diff --git a/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs b/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs index 2b4e495..2e4affc 100644 --- a/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs +++ b/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs @@ -42,7 +42,7 @@ public sealed class NpgsqlCrawlTargetRepository(NpgsqlDataSource source) : ICraw crawl_delay AS CrawlDelay, first_seen_at AS FirstSeenAt, last_probed_at AS LastProbedAt, discovered_from_game_id AS DiscoveredFromGameId, depth AS Depth, is_operator_seed AS IsOperatorSeed, submitted_at AS SubmittedAt, - discovered_via AS DiscoveredVia, mssp_route AS MsspRoute, + discovered_via AS DiscoveredVia, mssp_route AS MsspRoute, who_answers_at AS WhoAnswersAt, (SELECT f.value FROM game_field f WHERE f.game_id = crawl_target.game_id @@ -204,6 +204,16 @@ UPDATE crawl_target cancellationToken: ct)); } + public async Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct) + { + await using var connection = await source.OpenConnectionAsync(ct); + + await connection.ExecuteAsync(new CommandDefinition( + "UPDATE crawl_target SET who_answers_at = @at WHERE id = @id", + new { id, at = at?.ToUniversalTime() }, + cancellationToken: ct)); + } + /// /// Attaches the game this address turned out to be. /// @@ -262,6 +272,8 @@ private sealed class Row /// Text, not the enum, for the reason is. public string? MsspRoute { get; init; } + public DateTimeOffset? WhoAnswersAt { get; init; } + /// /// Nullable because the subquery has no row to read for a target not yet bound to a game, /// and null there means the same as true: an address that has proved nothing yet is asked @@ -291,6 +303,7 @@ private sealed class Row MsspCharset = MsspCharset, AwaitingCorroboration = AwaitingCorroboration ?? true, MsspRoute = MsspRouteSpelling.From(MsspRoute), + WhoAnswersAt = WhoAnswersAt, }; } } diff --git a/src/MUI.Discovery/Scheduling/CrawlTarget.cs b/src/MUI.Discovery/Scheduling/CrawlTarget.cs index 0ffeb4c..871e403 100644 --- a/src/MUI.Discovery/Scheduling/CrawlTarget.cs +++ b/src/MUI.Discovery/Scheduling/CrawlTarget.cs @@ -140,6 +140,12 @@ public sealed record CrawlTarget /// asks for and how it changes. /// public MsspRoute? MsspRoute { get; init; } + + /// + /// When a WHO typed at this address last came back with a count, or null when it is not + /// known to answer one (migration 0043). owns how it changes. + /// + public DateTimeOffset? WhoAnswersAt { get; init; } } /// @@ -193,5 +199,12 @@ Task RecordAttemptAsync( /// Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct); + /// + /// Records that this address answers WHO with a count, as of , or + /// with null that it no longer does. Written only on a change, like + /// . + /// + Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct); + Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct); } diff --git a/src/MUI.Discovery/Scheduling/WhoAnswers.cs b/src/MUI.Discovery/Scheduling/WhoAnswers.cs new file mode 100644 index 0000000..673f675 --- /dev/null +++ b/src/MUI.Discovery/Scheduling/WhoAnswers.cs @@ -0,0 +1,50 @@ +using MUI.Crawl; + +namespace MUI.Discovery; + +/// +/// What a probe teaches about whether an address answers a pre-login WHO +/// (crawl_target.who_answers_at, migration 0043). +/// +/// +/// A pure function of a target's memory and a , as +/// is, so it is testable without a socket or a database. +/// +public static class WhoAnswers +{ + /// + /// The new value of , or when + /// this probe taught nothing. + /// + /// + /// + /// A counted WHO sets it, dated by the probe, and only from unknown: re-dating it on every + /// crawl would be a write per probe to say what the row already says. + /// + /// + /// The login prompt taking WHO for a character name clears it, because that is the game + /// saying it has no pre-login WHO at this address any more, and typing it again every + /// crawl is exactly what the restraint in TelnetProbe.PublishedCountAsync exists to stop. + /// An unreadable answer does not: that is our parser meeting a dialect, not the game changing. + /// + /// + /// A failed dial teaches nothing, and neither does a session that never typed WHO. + /// + /// + public static DateTimeOffset? Learn(DateTimeOffset? current, ProbeResult result, DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(result); + + if (result.Outcome is not ProbeOutcome.Answered) + { + return current; + } + + if (result.Who.HasCount) + { + return current ?? now; + } + + return result.Who.Confidence is WhoConfidence.LoginPrompt ? null : current; + } +} diff --git a/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs b/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs index 7f5c5de..2eecb67 100644 --- a/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs +++ b/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs @@ -151,6 +151,29 @@ public async Task AKnownAnswererIsAskedBeforeWhoAndSparedTheWho() await Assert.That(result.Banner).Contains("Welcome to Old Realms"); } + /// + /// A stated count does not stand in for WHO at an address known to answer it. + /// + /// + /// tapestries.fur.com:2069 answers MSSP-REQUEST with PLAYERS = 26842, its + /// player objects, while its WHO counts a few hundred. The report spared it the + /// WHO, and the site published the database as the population. + /// + [Test] + public async Task AKnownAnswererKnownToAnswerWhoIsStillAskedWho() + { + await using var game = new OldRealms { AnswersRequest = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync( + game.Target with { PlaintextMssp = PlaintextMsspAsk.Ask, WhoAnswers = true }); + + await Assert.That(MsspReport.Last(result.Mssp, "PLAYERS")).IsEqualTo("4"); + await Assert.That(game.Sessions).Count().IsEqualTo(1); + await Assert.That(game.Sessions[0].First()).IsEqualTo(Request); + await Assert.That(game.Sessions[0]).Contains("WHO"); + await Assert.That(result.Who.Count).IsEqualTo(3); + } + [Test] public async Task AKnownAnswererThatFallsSilentIsRecordedAsSuch() { diff --git a/tests/MUI.Crawler.Tests/Crawl/WhoAnswersPostgresTests.cs b/tests/MUI.Crawler.Tests/Crawl/WhoAnswersPostgresTests.cs new file mode 100644 index 0000000..83f5c26 --- /dev/null +++ b/tests/MUI.Crawler.Tests/Crawl/WhoAnswersPostgresTests.cs @@ -0,0 +1,68 @@ +using Dapper; + +using MUI.Crawl; +using MUI.Crawler.Tests.Support; + +namespace MUI.Crawler.Tests; + +/// +/// What the crawl loop remembers about whether an address answers WHO +/// (crawl_target.who_answers_at), and that the next dial is told. +/// +public class WhoAnswersPostgresTests +{ + private const string Host = "tapestries.example.org"; + private const int Port = 2069; + + [Test] + public async Task AnAddressWhoseWhoCountedIsAskedWhoFromThenOn() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + var first = new ScriptedProbe(target => Probes.Answered( + host: target.Host, port: target.Port, who: new WhoReading(WhoConfidence.Count, 412))); + await CrawlCycles.Build(source, first, TimeProvider.System).RunAsync(); + + await Assert.That(first.Asked.Single().WhoAnswers).IsFalse(); + + await using var connection = await source.OpenConnectionAsync(); + + await Assert.That(await KnownAsync(connection)).IsTrue(); + + await DueAgainAsync(connection); + + var next = new ScriptedProbe(target => Probes.Answered( + host: target.Host, port: target.Port, who: new WhoReading(WhoConfidence.Count, 405))); + await CrawlCycles.Build(source, next, TimeProvider.System).RunAsync(); + + await Assert.That(next.Asked.Single().WhoAnswers).IsTrue(); + } + + [Test] + public async Task AnAddressWhoseLoginPromptAteWhoIsForgotten() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + await using var connection = await source.OpenConnectionAsync(); + await connection.ExecuteAsync("UPDATE crawl_target SET who_answers_at = now() - interval '1 day'"); + + var probe = new ScriptedProbe(target => Probes.Answered( + host: target.Host, port: target.Port, who: WhoReading.LoginPrompt)); + await CrawlCycles.Build(source, probe, TimeProvider.System).RunAsync(); + + await Assert.That(probe.Asked.Single().WhoAnswers).IsTrue(); + await Assert.That(await KnownAsync(connection)).IsFalse(); + } + + private static Task KnownAsync(Npgsql.NpgsqlConnection connection) => + connection.ExecuteScalarAsync("SELECT who_answers_at IS NOT NULL FROM crawl_target"); + + private static Task DueAgainAsync(Npgsql.NpgsqlConnection connection) => + connection.ExecuteAsync("UPDATE crawl_target SET next_probe_at = now() - interval '1 hour'"); +} diff --git a/tests/MUI.Crawler.Tests/Support/FakeStores.cs b/tests/MUI.Crawler.Tests/Support/FakeStores.cs index ec00405..8677e37 100644 --- a/tests/MUI.Crawler.Tests/Support/FakeStores.cs +++ b/tests/MUI.Crawler.Tests/Support/FakeStores.cs @@ -66,6 +66,8 @@ public Task RecordAttemptAsync( public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) => Task.CompletedTask; + public Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct) => Task.CompletedTask; + public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) => Task.CompletedTask; } diff --git a/tests/MUI.Discovery.Tests/Scheduling/WhoAnswersTests.cs b/tests/MUI.Discovery.Tests/Scheduling/WhoAnswersTests.cs new file mode 100644 index 0000000..4c78bef --- /dev/null +++ b/tests/MUI.Discovery.Tests/Scheduling/WhoAnswersTests.cs @@ -0,0 +1,49 @@ +using MUI.Crawl; +using MUI.Discovery.Tests.Support; + +namespace MUI.Discovery.Tests; + +/// +/// What a probe teaches about whether an address answers a pre-login WHO. +/// +public class WhoAnswersTests +{ + private static readonly DateTimeOffset Now = new(2026, 10, 7, 20, 0, 0, TimeSpan.Zero); + private static readonly DateTimeOffset Earlier = Now.AddDays(-3); + + [Test] + public async Task ACountedWhoIsRememberedAndDatedOnce() + { + var counted = ProbeResults.Answered(who: new WhoReading(WhoConfidence.Count, 412)); + + await Assert.That(WhoAnswers.Learn(null, counted, Now)).IsEqualTo(Now); + + // Already known: the row already says it, so nothing is re-dated. + await Assert.That(WhoAnswers.Learn(Earlier, counted, Now)).IsEqualTo(Earlier); + } + + [Test] + public async Task TheLoginPromptTakingWhoForANameForgetsIt() + { + var ate = ProbeResults.Answered(who: WhoReading.LoginPrompt); + + await Assert.That(WhoAnswers.Learn(Earlier, ate, Now)).IsNull(); + } + + [Test] + public async Task AnUnreadableWhoIsOurParserAndChangesNothing() + { + var unreadable = ProbeResults.Answered(who: WhoReading.Unreadable); + + await Assert.That(WhoAnswers.Learn(Earlier, unreadable, Now)).IsEqualTo(Earlier); + await Assert.That(WhoAnswers.Learn(null, unreadable, Now)).IsNull(); + } + + [Test] + public async Task ASessionThatNeverAskedOrNeverConnectedTeachesNothing() + { + await Assert.That(WhoAnswers.Learn(Earlier, ProbeResults.Answered(), Now)).IsEqualTo(Earlier); + await Assert.That(WhoAnswers.Learn(Earlier, ProbeResults.Failed(), Now)).IsEqualTo(Earlier); + await Assert.That(WhoAnswers.Learn(null, ProbeResults.Failed(), Now)).IsNull(); + } +} diff --git a/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs b/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs index dd9557f..907464d 100644 --- a/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs +++ b/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs @@ -78,6 +78,13 @@ public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, C return Task.CompletedTask; } + public Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct) + { + Replace(id, target => target with { WhoAnswersAt = at }); + + return Task.CompletedTask; + } + public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) { Replace(id, target => target.GameId is null ? target with { GameId = gameId } : target); diff --git a/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs b/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs index c3382bb..163d9a4 100644 --- a/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs +++ b/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs @@ -183,6 +183,9 @@ public Task RecordTransportAsync(Guid id, bool useTls, CancellationToken ct) => public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) => throw new NotSupportedException(); + public Task RecordWhoAnswersAsync(Guid id, DateTimeOffset? at, CancellationToken ct) => + throw new NotSupportedException(); + public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) => throw new NotSupportedException(); }