From 4ef4afb9b78091165820d43ee3b2c0e9746b26a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 6 Oct 2026 15:50:29 +0000 Subject: [PATCH] Try the plaintext MSSP-REQUEST once per address that never gave option 70 TelnetNegotiationCore 4.0 ships MSSPPlaintextProtocol, which is what the probe was waiting for. The crawl loop now remembers, per address, how MSSP arrives (crawl_target.mssp_route, migration 0042): - telnet: reported over option 70. Never sent MSSP-REQUEST. Backfilled for every game that already has mssp or mssp_roster fields. - NULL: unknown. After an ordinary session that got no MSSP, the probe makes one short second dial that sends MSSP-REQUEST and nothing else, so a name prompt that turns it into a password prompt never gets the measuring session's WHO typed into it. - plaintext: answered. Asked in-session every crawl, before WHO, so a PLAYERS in the reply spares the game the WHO. - none: did not answer. Never asked again. A plaintext report is stored as mssp like an option-70 one, and is not noted as an option-70 negotiation in the handshake fields. MsspRoutes holds both the ask and the transition, as pure functions. mui-probe gains --mssp-request and --mssp-request=ask for trying it by hand. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01YK1nuQCYc9TnnDnWEjcNpS --- CLAUDE.md | 7 + docs/codebase-survey-2026-07-30.md | 11 +- migrations/0042_mssp_route.sql | 59 +++ src/MUI.Crawl/Telnet/ProbeOptions.cs | 17 +- src/MUI.Crawl/Telnet/ProbeResult.cs | 44 ++- src/MUI.Crawl/Telnet/Probing.cs | 54 +++ src/MUI.Crawl/Telnet/TelnetProbe.cs | 295 ++++++++++++++- src/MUI.Crawler/Crawl/CrawlCycle.cs | 20 ++ src/MUI.Crawler/Crawl/FieldObservations.cs | 5 +- .../NpgsqlCrawlTargetRepository.cs | 49 ++- src/MUI.Discovery/Scheduling/CrawlTarget.cs | 21 ++ src/MUI.Discovery/Scheduling/MsspRoutes.cs | 93 +++++ src/MUI.Probe/Program.cs | 14 + .../Telnet/PlaintextMsspTests.cs | 339 ++++++++++++++++++ .../Telnet/ProbeRestraintTests.cs | 9 +- .../Crawl/MsspRoutePostgresTests.cs | 170 +++++++++ tests/MUI.Crawler.Tests/Support/FakeStores.cs | 3 + .../Scheduling/MsspRoutesTests.cs | 91 +++++ .../Support/InMemoryCrawlTargetRepository.cs | 7 + .../MUI.Web.Tests/StoredCrawlerPulseTests.cs | 3 + 20 files changed, 1278 insertions(+), 33 deletions(-) create mode 100644 migrations/0042_mssp_route.sql create mode 100644 src/MUI.Discovery/Scheduling/MsspRoutes.cs create mode 100644 tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs create mode 100644 tests/MUI.Crawler.Tests/Crawl/MsspRoutePostgresTests.cs create mode 100644 tests/MUI.Discovery.Tests/Scheduling/MsspRoutesTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 6f66ecc6..250361fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,6 +93,13 @@ our host is unreachable and perfectly alive. typed at the family above, every crawl** — issue #182 is where that is being thought about, and it is not the same problem: there is nothing they would publish, so "not asking must imply publishing" has nothing to hold on to. +- **Type `MSSP-REQUEST` at an address that has reported over option 70, or that left its trial + unanswered.** The plaintext form is text at a login prompt, and eight of twenty surveyed games read + it as a character name. `crawl_target.mssp_route` (migration 0042) remembers each address's answer + and `MsspRoutes` is the one place that reads it: an unknown address gets **one** trial, on a dial + of its own after the measurement (`PlaintextMsspAsk.Trial`), so a name prompt that turns the request + into a password prompt never gets that session's `WHO` typed into it. A plaintext report is stored + as `mssp` like any other report, and is never noted as an option-70 negotiation. - **Read a roster as a total, or fold it into `mssp`.** `MsspCountKind.Roster` is a **floor**: `tdome.nukefire.org:4000` states `PLAYERS = 70` and names sixty-nine, stably, because every codebase has someone it does not show. It gets its own `mssp_roster` source so a reader can tell diff --git a/docs/codebase-survey-2026-07-30.md b/docs/codebase-survey-2026-07-30.md index 7ca61f45..306d896d 100644 --- a/docs/codebase-survey-2026-07-30.md +++ b/docs/codebase-survey-2026-07-30.md @@ -222,10 +222,13 @@ the entire time. ## The plaintext `MSSP-REQUEST` form — 3 answered of 20, and 8 read it as a name -**Measured, and deliberately not implemented here.** The plaintext form belongs in -TelnetNegotiationCore, where it is filed as **issue #61**; `CLAUDE.md`'s rule is that a gap in that -library is a PR rather than a compensating hack in this repository. This section is the evidence for -whoever writes that PR — the probe carries none of it. +**Measured here; now asked, narrowly.** The plaintext form belongs in TelnetNegotiationCore, where +it was filed as **issue #61** and shipped as `MSSPPlaintextProtocol` (4.0). The probe now uses it under +the policy this section argues for: never to an address that has reported over option 70, once to +every other address on a dial of its own after the measurement, every session only to an address +that answered, and never again to one that did not (`crawl_target.mssp_route`, migration 0042; +`MsspRoutes`). The cost below is why it is once and on its own dial: a name prompt that turns the +request into a password prompt must not get the measuring session's `WHO` typed into it. Twenty games were sent the literal line `MSSP-REQUEST` at their login screen. Three answered with a well-formed `MSSP-REPLY-START` / tab-separated / `MSSP-REPLY-END` report: diff --git a/migrations/0042_mssp_route.sql b/migrations/0042_mssp_route.sql new file mode 100644 index 00000000..0f45a3af --- /dev/null +++ b/migrations/0042_mssp_route.sql @@ -0,0 +1,59 @@ +-- crawl_target.mssp_route: which way, if any, this address hands over its MSSP report. +-- +-- The plaintext form — the line `MSSP-REQUEST` typed at the connect screen, answered between +-- `MSSP-REPLY-START` and `MSSP-REPLY-END` — is the SMAUG family's, and TelnetNegotiationCore 4.0 +-- carries it as MSSPPlaintextProtocol. Unlike `IAC DO 70`, which a server without MSSP ignores, it is +-- text at a stranger's login prompt: of twenty games sent it in docs/codebase-survey-2026-07-30.md, +-- three answered and eight read it as a character name. So it is asked at most once of an address +-- that has not answered it, and never of one that reports the proper way. This column is how the +-- crawl loop remembers which is which: +-- +-- NULL nothing known yet. The next session that reaches the game without an option-70 +-- report is followed by one short dial that asks MSSP-REQUEST and nothing else. +-- telnet a report has arrived over option 70. Never asked the plaintext form, ever: the user +-- rule this was built from is "only for games that have not already given us MSSP the +-- proper way". +-- plaintext answered MSSP-REQUEST. Asked again every session, at the connect screen, because it +-- is the only route this address has. +-- none sent MSSP-REQUEST and did not answer it. Never asked again. +-- +-- A measurement of them, unlike crawl_refusal, which is our note about our own decision: whether a +-- server answers a line of the MSSP specification is a fact about the server. It is still kept off +-- every public surface. The game page's MSSP fields carry the report itself, whichever route brought +-- it; how it travelled says nothing a reader needs. +-- +-- Per address, because the probe dials addresses. A game reachable on two ports is asked on each, +-- which is right: nothing makes two listeners run the same code. +-- +-- No BEGIN/COMMIT: MigrationRunner opens its own transaction per script and writes the ledger entry +-- inside it. + +ALTER TABLE crawl_target + ADD COLUMN mssp_route text, + ADD COLUMN mssp_route_at timestamptz, + + ADD CONSTRAINT crawl_target_mssp_route_vocabulary CHECK ( + mssp_route IN ('telnet', 'plaintext', 'none')), + + -- Set together: a route nobody can date is one nobody can judge the age of. + ADD CONSTRAINT crawl_target_mssp_route_is_dated CHECK ( + (mssp_route IS NULL) = (mssp_route_at IS NULL)); + +COMMENT ON COLUMN crawl_target.mssp_route IS + 'How this address hands over MSSP: telnet (option 70, so the plaintext MSSP-REQUEST is never ' + 'sent), plaintext (answers MSSP-REQUEST, asked every session), none (did not answer it, never ' + 'asked again), or NULL (not yet known; gets one trial).'; + +-- Every game that has already reported over option 70 is excused the trial. Until this migration +-- nothing could put a value under these two sources except an option-70 report, so their presence +-- is exactly "has given us MSSP the proper way". Dated now rather than back-dated: this is when the +-- crawl loop learnt it, and the rows' own first_seen_at would claim a knowledge of this column that +-- nobody had. +UPDATE crawl_target t + SET mssp_route = 'telnet', + mssp_route_at = now() + WHERE EXISTS ( + SELECT 1 + FROM game_field f + WHERE f.game_id = t.game_id + AND f.source IN ('mssp', 'mssp_roster')); diff --git a/src/MUI.Crawl/Telnet/ProbeOptions.cs b/src/MUI.Crawl/Telnet/ProbeOptions.cs index b4859771..da13cc4b 100644 --- a/src/MUI.Crawl/Telnet/ProbeOptions.cs +++ b/src/MUI.Crawl/Telnet/ProbeOptions.cs @@ -160,11 +160,18 @@ public sealed record ProbeOptions /// public int MaxPromptRounds { get; init; } = 4; - // Deliberately no option here for the plaintext MSSP-REQUEST form — it belongs in - // TelnetNegotiationCore, and a compensating implementation here would duplicate a first-party - // dependency and then have to be deleted once that lands. Every surveyed game that answered the - // plaintext form also answered telnet option 70, so it reached nothing extra; several others read - // the bare request as a character name and burned a login attempt on it. + /// + /// How long a plaintext MSSP-REQUEST waits for MSSP-REPLY-END before the game is + /// taken not to answer it. + /// + /// + /// Handed to TelnetNegotiationCore's MSSPPlaintextProtocol.WithReplyTimeout. Ten seconds is + /// the library's own default and Grapevine's crawler's window. Every reply measured + /// (docs/codebase-survey-2026-07-30.md) arrived well inside it; a server that has not + /// started one by then has read the request as a character name. Only ever spent where + /// says to ask at all. + /// + public TimeSpan PlaintextMsspGrace { get; init; } = TimeSpan.FromSeconds(10); /// /// Ceiling on a single subnegotiation payload, handed to TelnetNegotiationCore's diff --git a/src/MUI.Crawl/Telnet/ProbeResult.cs b/src/MUI.Crawl/Telnet/ProbeResult.cs index 17a14d6f..18b39d93 100644 --- a/src/MUI.Crawl/Telnet/ProbeResult.cs +++ b/src/MUI.Crawl/Telnet/ProbeResult.cs @@ -141,6 +141,17 @@ public IReadOnlyList MsspValues(string variable) => /// Which transport produced . Part of the value's provenance. public MsspTransport MsspTransport { get; init; } = MsspTransport.None; + /// + /// What came of the plaintext MSSP-REQUEST, when + /// had it sent. + /// + /// + /// Kept apart from , which an unanswered request never changes: a game + /// that ignored the plaintext line has the same it had + /// before it was asked. Read by the crawl loop, to decide whether to ask again. + /// + public PlaintextMsspOutcome PlaintextMssp { get; init; } = PlaintextMsspOutcome.NotAsked; + public FailureDetail? Failure { get; init; } /// @@ -168,7 +179,10 @@ public IReadOnlyList MsspValues(string variable) => /// public enum MsspOutcome { - /// The server never offered MSSP and did not answer the plaintext request. + /// + /// No report arrived: the server never offered MSSP and, if it was sent the plaintext request, + /// did not answer that either. + /// NotOffered, /// A report arrived and was parsed. It may still be empty, which is the server's answer. @@ -194,13 +208,35 @@ public enum MsspTransport /// The plaintext MSSP-REQUEST reply, delimited by START/END markers. /// - /// Nothing produces this yet, deliberately — the plaintext form belongs in - /// TelnetNegotiationCore (first-party), and implementing it here would duplicate then have to be - /// deleted. The member stays because spec §6.4 describes both routes. + /// Read by TelnetNegotiationCore's MSSPPlaintextProtocol; this repository only decides + /// when to ask (). /// PlaintextRequest, } +/// What came of a plaintext MSSP-REQUEST. +public enum PlaintextMsspOutcome +{ + /// + /// The line was never sent: nobody asked for it, the session negotiated option 70, or the + /// connection was gone first. Says nothing about the game. + /// + NotAsked, + + /// + /// The game answered with an MSSP-REPLY-START to MSSP-REPLY-END block, including + /// one too large to keep, which is still an answer. + /// + Answered, + + /// + /// The line went out and no reply came back before + /// ran out or the game hung up. A measurement of the game, and the one that stops it being asked + /// again. + /// + Unanswered, +} + public enum ProbeOutcome { Answered, diff --git a/src/MUI.Crawl/Telnet/Probing.cs b/src/MUI.Crawl/Telnet/Probing.cs index a071a921..4c6f857b 100644 --- a/src/MUI.Crawl/Telnet/Probing.cs +++ b/src/MUI.Crawl/Telnet/Probing.cs @@ -82,9 +82,63 @@ public sealed record ProbeTarget(string Host, int Port) /// public bool UseTls { get; init; } + /// + /// Whether, and how, to send the plaintext MSSP-REQUEST line this time. + /// + /// + /// + /// Defaults to , the opposite direction from + /// and for the opposite reason: IAC DO 70 is ignored by + /// a server without MSSP, but this is a line of text at somebody's login prompt, and eight of the + /// twenty games surveyed read it as a character name (docs/codebase-survey-2026-07-30.md). + /// A caller that cannot say what this address has done before does not type it. + /// + /// + /// Only the crawl loop answers it, from what the address did before (crawl_target.mssp_route, + /// see MsspRoutes): an address that has ever reported over option 70, or failed to answer + /// this line, is never asked; one that answered it is asked every session; one that has done + /// neither gets one . + /// + /// + public PlaintextMsspAsk PlaintextMssp { get; init; } = PlaintextMsspAsk.Never; + public override string ToString() => Host.Contains(':') ? $"[{Host}]:{Port}" : $"{Host}:{Port}"; } +/// What a probe does about the plaintext MSSP-REQUEST form. +/// +/// The plaintext form is the SMAUG family's (spec §6.4); TelnetNegotiationCore carries it as +/// MSSPPlaintextProtocol. It is never sent in a session that negotiated option 70, whatever +/// this says: a game already reporting the proper way has nothing to add by the older one. +/// +public enum PlaintextMsspAsk +{ + /// Not sent. + Never, + + /// + /// Sent once, on a second short dial of its own, after an ordinary session that got no MSSP. + /// + /// + /// A dial of its own because the measurement session must not pay for the experiment. A game + /// that reads the line as a character name moves on to a password prompt, and every command after + /// it, WHO included, is then typed into the wrong question. That would publish our + /// experiment as an unreadable WHO (rule 5). The trial dial asks for nothing else and hangs + /// up. + /// + Trial, + + /// + /// Sent in the ordinary session, at the connect screen and before WHO, because this address + /// has answered it before. + /// + /// + /// Before WHO on purpose: the report usually carries PLAYERS, which is a count + /// PublishedCountAsync accepts in place of typing WHO at all. + /// + Ask, +} + /// /// What carried a probe session. Part of the result's provenance, and the whole of what /// EndpointKind.Tls is written from. diff --git a/src/MUI.Crawl/Telnet/TelnetProbe.cs b/src/MUI.Crawl/Telnet/TelnetProbe.cs index 2f98b91b..7a9601f3 100644 --- a/src/MUI.Crawl/Telnet/TelnetProbe.cs +++ b/src/MUI.Crawl/Telnet/TelnetProbe.cs @@ -43,8 +43,9 @@ public sealed class TelnetProbe(ProbeOptions? options = null, ILogger? logger = /// /// /// The bare line terminator sent between the banner and WHO is not on this list — it's not - /// a command, it carries no text. MSSP-REQUEST is also absent by design (see - /// ): MSSP is asked for by negotiation, which a server without it ignores. + /// a command, it carries no text. MSSP-REQUEST is not on it either: it is sent only by + /// TelnetNegotiationCore's MSSPPlaintextProtocol, only where + /// says this address may be asked, and never in a session that negotiated option 70. /// The other thing that goes on the wire is a classified prompt answer, which is not a command /// either and is bounded separately by . /// @@ -83,6 +84,27 @@ public static bool IsPermittedPromptAnswer(string? answer) => private readonly ILogger _logger = logger ?? NullLogger.Instance; public async Task ProbeAsync(ProbeTarget target, CancellationToken cancellationToken = default) + { + var result = await MeasureAsync(target, cancellationToken); + + // The plaintext trial runs after the measurement and never inside it — see + // PlaintextMsspAsk.Trial. Only for a session that reached the game and got no MSSP at all: + // a game that negotiated option 70, even one whose report never came or was too large to + // keep, is reporting the proper way and has nothing to add by the older one. + return target.PlaintextMssp is PlaintextMsspAsk.Trial + && result is + { + Outcome: ProbeOutcome.Answered, + MsspOutcome: MsspOutcome.NotOffered, + PlaintextMssp: PlaintextMsspOutcome.NotAsked, + } + && !result.OfferedOptions.Contains("MSSP") + ? await TrialPlaintextMsspAsync(target, result, cancellationToken) + : result; + } + + /// The measurement: one session, and the other transport when the first learnt nothing. + private async Task MeasureAsync(ProbeTarget target, CancellationToken cancellationToken) { var session = await SessionAsync( target, @@ -270,7 +292,7 @@ bool Heard() } var prompts = new PromptSink(lines); - var builder = Build(seen, lines, prompts); + var builder = Build(seen, lines, prompts, target.PlaintextMssp is PlaintextMsspAsk.Ask); // The stream overload and the TcpClient one differ only in where the pipe comes from, so // everything above the socket — negotiation, MSSP, the WHO parser, the encoding decision @@ -323,6 +345,17 @@ int Arrived() // handed to the wrong parameter by a future edit that reorders phases. var cursors = new PhaseCursors { Banner = Arrived() }; + // An address that has answered the plaintext form before is asked it here: after the + // connect screen has settled, so the request lands at the name prompt the SMAUG family + // reads it at, and before WHO, so a PLAYERS in the reply can spare the game the WHO. + // Whatever the line provokes besides the reply (the reply's own leading blank line, a + // re-printed prompt) arrives after cursors.Banner and before cursors.Flush, the slice + // that is thrown away for the same reason the flush line's reaction is: it is a reaction + // to text we sent, not the game's screen. + var plaintext = target.PlaintextMssp is PlaintextMsspAsk.Ask + ? await AskPlaintextMsspAsync(telnet, client, seen, target, budget.Token) + : PlaintextMsspOutcome.NotAsked; + // A who's-online menu option is different from every category the prompt loop above // answers: selecting it doesn't reveal a second screen behind this one — for every real // game measured (BatMUD, ZombieMUD, discworld.starturtle.net) the menu already settled @@ -365,7 +398,9 @@ int Arrived() // has already done the flushing. Computed here rather than read off // whoAlreadyAnswered inside, so that flag keeps its one meaning: if WHO ever comes to // be answered by a route that sends nothing, the flush must not silently go with it. - alreadyFlushed: answeredAPrompt || whoFromMenu is not null, + alreadyFlushed: answeredAPrompt + || whoFromMenu is not null + || plaintext is not PlaintextMsspOutcome.NotAsked, whoAlreadyAnswered: whoFromMenu is not null, markAsked: () => asked = true, target, @@ -385,7 +420,8 @@ int Arrived() return new Session( BuildAnsweredResult( target, observedAt, started, seen, reading, cursors, - whoFromMenu, whoFromMenuShape, asked, published, telnet.CurrentEncoding, transport), + whoFromMenu, whoFromMenuShape, asked, published, telnet.CurrentEncoding, transport, + plaintext), Heard()); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -444,6 +480,200 @@ or ObjectDisposedException } } + /// + /// Sends the plaintext MSSP-REQUEST and waits for the reply, unless this session has + /// already shown it has no need of one. + /// + /// + /// + /// The exchange itself is TelnetNegotiationCore's (MSSPPlaintextProtocol): it frames the + /// request, reads the reply off the line stream so none of it reaches lines as screen, and + /// hands the report to the same OnMSSP callback option 70 uses. This method decides only + /// whether to ask, and what the answer says about the game. + /// + /// + /// covers a hang-up after the line went out as well + /// as silence: a server that ends the session on an unknown name has answered the question as + /// plainly as one that says Illegal name. A socket already gone beforehand is + /// , because then nothing was asked. + /// + /// + private async Task AskPlaintextMsspAsync( + TelnetInterpreter telnet, + TcpClient client, + Observations seen, + ProbeTarget target, + CancellationToken cancellationToken) + { + if (seen.Supported.Contains("MSSP") + || seen.MsspOutcome is not MsspOutcome.NotOffered + || !Live(client) + || telnet.PluginManager?.GetPlugin() is not { } plaintext) + { + return PlaintextMsspOutcome.NotAsked; + } + + seen.AskingPlaintext = true; + + try + { + var report = await plaintext.RequestReportAsync(cancellationToken); + + return report is not null || seen.MsspOutcome is MsspOutcome.RejectedTooLarge + ? PlaintextMsspOutcome.Answered + : PlaintextMsspOutcome.Unanswered; + } + catch (Exception error) when (HungUp(error)) + { + // The write itself failed, so the line never reached them. + _logger.LogDebug( + "{Host}:{Port} was gone before MSSP-REQUEST could be sent ({Error})", + target.Host, target.Port, error.Message); + return PlaintextMsspOutcome.NotAsked; + } + finally + { + seen.AskingPlaintext = false; + } + } + + /// + /// The second, short dial that asks an address the plaintext form for the first time: the + /// connect screen, any pre-screen prompts, MSSP-REQUEST, and nothing else. + /// + /// + /// + /// is returned with the trial's outcome on it and, when the game + /// answered, its report — the same report a third session would have read, minutes earlier than + /// waiting a whole crawl interval for it. + /// + /// + /// The trial can only add to the measurement, never cost it. Anything that goes wrong on + /// this dial — refused, reset, a handshake that will not complete, a defect here — returns + /// unchanged with , which + /// the crawl loop reads as "still to try". Our caller going away is the one exception, rethrown + /// for the same reason rethrows it. + /// + /// + private async Task TrialPlaintextMsspAsync( + ProbeTarget target, + ProbeResult measured, + CancellationToken cancellationToken) + { + using var budget = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + budget.CancelAfter(_options.Timeout); + + var lines = new List(); + var seen = new Observations(); + + using var client = new TcpClient(); + SslStream? tunnel = null; + + try + { + await (target.Addresses.Count > 0 + ? client.ConnectAsync([.. target.Addresses], target.Port, budget.Token) + : client.ConnectAsync(target.Host, target.Port, budget.Token)); + + // The door the measurement just came through, not the one the target was dialled with: + // the measurement may have had to try the other one. + if (measured.Transport is ProbeTransport.Tls) + { + tunnel = await HandshakeAsync(client, target, budget.Token); + } + + var prompts = new PromptSink(lines); + var builder = Build(seen, lines, prompts, plaintextMssp: true); + var built = tunnel is null + ? await builder.BuildAndStartAsync(client, budget.Token) + : await builder.BuildAndStartAsync(tunnel, budget.Token); + + prompts.Reads(built.Interpreter); + await using var telnet = built.Interpreter; + _ = ObserveReadLoopAsync(built.ReadTask, target); + + int Arrived() + { + lock (lines) + { + return lines.Count; + } + } + + // The same approach to the name prompt the measurement takes, so the request lands where + // a SMAUG-family nanny reads it — not on a colour question or a press-enter gate. + await SettleInitialBannerAsync(telnet, Arrived, lines, budget.Token); + await AnswerPromptsAsync(telnet, Arrived, lines, client, budget.Token); + + var outcome = await AskPlaintextMsspAsync(telnet, client, seen, target, budget.Token); + + _logger.LogDebug( + "{Host}:{Port} plaintext MSSP-REQUEST: {Outcome}", target.Host, target.Port, outcome); + + if (outcome is not PlaintextMsspOutcome.Answered) + { + return measured with { PlaintextMssp = outcome }; + } + + if (seen.MsspOutcome is MsspOutcome.RejectedTooLarge) + { + return measured with + { + PlaintextMssp = outcome, + MsspOutcome = MsspOutcome.RejectedTooLarge, + MsspBytesRejected = seen.MsspRejectedBytes, + }; + } + + // The report's encoding is decided the way the measurement decides it, against the screen + // it arrived beside — here, this dial's. + WireReading reading; + lock (lines) + { + reading = WireEncoding.Read( + lines, target.Charset, MsspReport.RawValues(seen.Mssp), target.MsspCharset); + } + + return measured with + { + PlaintextMssp = outcome, + Mssp = MsspReport.From(seen.Mssp, reading.MsspEncoding), + MsspOutcome = MsspOutcome.Received, + MsspTransport = MsspTransport.PlaintextRequest, + }; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception error) + { + if (HungUp(error) + || error is SocketException or OperationCanceledException or AuthenticationException) + { + _logger.LogDebug( + "{Host}:{Port} plaintext MSSP trial did not get as far as asking ({Error})", + target.Host, target.Port, error.Message); + } + else + { + _logger.LogWarning( + error, + "{Host}:{Port} plaintext MSSP trial failed; the measurement stands", + target.Host, target.Port); + } + + return measured; + } + finally + { + if (tunnel is not null) + { + await tunnel.DisposeAsync(); + } + } + } + /// /// Opens the TLS tunnel this session will speak telnet inside. /// @@ -635,8 +865,7 @@ async Task AskAsync(string command, int baseline, TimeSpan grace, Action? s try { // MSDP's request vocabulary — SEND, REPORT, LIST, RESET, UNREPORT — has no plaintext - // form, so it is not on PermittedCommands for the same reason MSSP-REQUEST is not: it - // is asked for by protocol, not by typing. Gated on TelnetNegotiationCore 2.9.0's + // form, so it is not on PermittedCommands: it is asked for by protocol, not by typing. Gated on TelnetNegotiationCore 2.9.0's // IsNegotiated (see Watched.Msdp), which reflects the peer's real WILL/DO acceptance — // unlike the pre-2.9.0 OnEnabledAsync, which was true from plugin construction // regardless of the wire (TelnetNegotiationCore#85). By this point in the probe the @@ -883,7 +1112,8 @@ private static ProbeResult BuildAnsweredResult( bool asked, int? published, Encoding? negotiatedEncoding, - ProbeTransport transport) + ProbeTransport transport, + PlaintextMsspOutcome plaintext) { var read = reading.Lines; @@ -916,7 +1146,7 @@ private static ProbeResult BuildAnsweredResult( seen.Charset ??= negotiatedEncoding.WebName; } - var viaOption = seen.MsspOutcome is MsspOutcome.Received; + var received = seen.MsspOutcome is MsspOutcome.Received; return new ProbeResult { @@ -946,10 +1176,11 @@ private static ProbeResult BuildAnsweredResult( // why not asking must imply publishing. Second, not first: the session-wide charset // decision is made above and a screen this decode reads better is read better. BannerPlayerCount = BannerCount.Find(banner) ?? published, - Mssp = viaOption ? MsspReport.From(seen.Mssp, reading.MsspEncoding) : MsspReport.Empty, + Mssp = received ? MsspReport.From(seen.Mssp, reading.MsspEncoding) : MsspReport.Empty, MsspOutcome = seen.MsspOutcome, MsspBytesRejected = seen.MsspRejectedBytes, - MsspTransport = viaOption ? MsspTransport.TelnetOption70 : MsspTransport.None, + MsspTransport = received ? seen.MsspTransport : MsspTransport.None, + PlaintextMssp = plaintext, Transport = transport, Elapsed = Stopwatch.GetElapsedTime(started), }; @@ -1221,7 +1452,11 @@ public ValueTask TakeAsync() /// separate statement apiece; the TelnetNegotiationCore plugin builders are already self-typed /// fluent, so this is the shape the library wants, not a new abstraction layered over it. /// - private TelnetInterpreterBuilder Build(Observations seen, List lines, PromptSink prompts) + private TelnetInterpreterBuilder Build( + Observations seen, + List lines, + PromptSink prompts, + bool plaintextMssp) { void Note(string protocol) => seen.Note(protocol); @@ -1231,7 +1466,20 @@ private TelnetInterpreterBuilder Build(Observations seen, List lines, Pr { seen.Mssp = config; seen.MsspOutcome = MsspOutcome.Received; - Note("MSSP"); + + // The plaintext reply arrives through this same callback, and it is not option 70. + // Supported means "observed active in the handshake", so noting MSSP for a line of + // text would publish a negotiation that never happened as a measured capability. + if (config.Source is MSSPSource.Plaintext) + { + seen.MsspTransport = MsspTransport.PlaintextRequest; + } + else + { + seen.MsspTransport = MsspTransport.TelnetOption70; + Note("MSSP"); + } + return ValueTask.CompletedTask; }) // 2.7.0 drops an oversized report whole rather than truncating it. Kept as its own @@ -1240,7 +1488,13 @@ private TelnetInterpreterBuilder Build(Observations seen, List lines, Pr { seen.MsspOutcome = MsspOutcome.RejectedTooLarge; seen.MsspRejectedBytes = (int)Math.Min(sizes.Item1, int.MaxValue); - Note("MSSP"); + + // The callback does not say which route overran, so the request in flight does. + if (!seen.AskingPlaintext) + { + Note("MSSP"); + } + return ValueTask.CompletedTask; }); @@ -1321,7 +1575,7 @@ private TelnetInterpreterBuilder Build(Observations seen, List lines, Pr // of the TTYPE list. var terminalType = new Watched.TerminalType(Note).WithTerminalTypes([.. _options.TerminalTypes]); - return new TelnetInterpreterBuilder() + var builder = new TelnetInterpreterBuilder() .UseMode(TelnetInterpreter.TelnetMode.Client) .UseLogger(_logger) .WithClientIdentity(_options.TerminalTypes.Count > 0 ? _options.TerminalTypes[0] : "MUINDEX-CRAWLER") @@ -1388,6 +1642,13 @@ private TelnetInterpreterBuilder Build(Observations seen, List lines, Pr .AddPlugin(new PacketPatchProtocol() .WithHoldTime(_options.PromptHold) .OnPrompt(prompts.TakeAsync)); + + // Registered only when this session may ask. On a client the plugin sends nothing until + // RequestReportAsync is called, so this is belt and braces: a session that was not told to + // ask has no way to. + return plaintextMssp + ? builder.AddPlugin(new MSSPPlaintextProtocol().WithReplyTimeout(_options.PlaintextMsspGrace)) + : builder; } /// @@ -1447,6 +1708,10 @@ private sealed class Observations public MSSPConfig? Mssp; public MsspOutcome MsspOutcome = MsspOutcome.NotOffered; + public MsspTransport MsspTransport = MsspTransport.None; + + /// Set while a plaintext MSSP-REQUEST waits for its reply. + public volatile bool AskingPlaintext; public int? MsspRejectedBytes; public string? Charset; public bool Prompts; diff --git a/src/MUI.Crawler/Crawl/CrawlCycle.cs b/src/MUI.Crawler/Crawl/CrawlCycle.cs index ccb13670..2498ff54 100644 --- a/src/MUI.Crawler/Crawl/CrawlCycle.cs +++ b/src/MUI.Crawler/Crawl/CrawlCycle.cs @@ -259,6 +259,10 @@ private async Task AttemptAsync(CrawlTarget target, CancellationToken c // known to be behind a handshake — and the probe overrules it either way if the game // has since moved, which is what keeps a stale flag from going dark for ever. UseTls = target.UseTls, + + // Whether to type the plaintext MSSP-REQUEST, from what this address did with it + // before. See MsspRoutes for both halves of that memory. + PlaintextMssp = MsspRoutes.Ask(target.MsspRoute), }, budget.Token); @@ -498,6 +502,22 @@ await targets.RecordAttemptAsync( { await targets.RecordTransportAsync(target.Id, transport, cancellationToken); } + + // The same shape for the MSSP route: written only when this probe taught something new. + // MsspRoutes.Learn already returns the row unchanged for a dial that failed. + if (MsspRoutes.Learn(target.MsspRoute, result) is var route && route != target.MsspRoute) + { + await targets.RecordMsspRouteAsync(target.Id, route, now, cancellationToken); + + if (route is MsspRoute.Plaintext or MsspRoute.None) + { + logger?.LogInformation( + "{Host}:{Port} {Answer} the plaintext MSSP-REQUEST", + target.Host, + target.Port, + route is MsspRoute.Plaintext ? "answered" : "did not answer"); + } + } } /// diff --git a/src/MUI.Crawler/Crawl/FieldObservations.cs b/src/MUI.Crawler/Crawl/FieldObservations.cs index b5abecd2..5dd8938f 100644 --- a/src/MUI.Crawler/Crawl/FieldObservations.cs +++ b/src/MUI.Crawler/Crawl/FieldObservations.cs @@ -181,7 +181,10 @@ private static IEnumerable Parsed( /// (ProbeOptions.MsspSettleGrace; found flapping capability.mssp.measured true/false /// in production for DIKU-family games that answer MSSP cleanly on every direct probe). Servers /// that only respond to IAC DO MSSP rather than advertising it themselves will appear as - /// until TNC gains a client-side request. + /// . A report that arrived by the plaintext + /// MSSP-REQUEST instead is , so it writes no negative + /// either: the game handed us MSSP, and publishing "no MSSP" beside its report would be false, + /// while it never offered option 70, so no true is written for the handshake. /// is not that case and is recorded as present: the /// server offered, we just chose not to hold the reply (§6.4). A protocol the library named that /// isn't in is still recorded — the registry isn't a gate on diff --git a/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs b/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs index 1723e5c3..2b4e4955 100644 --- a/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs +++ b/src/MUI.Crawler/Persistence/NpgsqlCrawlTargetRepository.cs @@ -42,7 +42,7 @@ public sealed class NpgsqlCrawlTargetRepository(NpgsqlDataSource source) : ICraw crawl_delay AS CrawlDelay, first_seen_at AS FirstSeenAt, last_probed_at AS LastProbedAt, discovered_from_game_id AS DiscoveredFromGameId, depth AS Depth, is_operator_seed AS IsOperatorSeed, submitted_at AS SubmittedAt, - discovered_via AS DiscoveredVia, + discovered_via AS DiscoveredVia, mssp_route AS MsspRoute, (SELECT f.value FROM game_field f WHERE f.game_id = crawl_target.game_id @@ -189,6 +189,21 @@ await connection.ExecuteAsync(new CommandDefinition( cancellationToken: ct)); } + public async Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) + { + await using var connection = await source.OpenConnectionAsync(ct); + + await connection.ExecuteAsync(new CommandDefinition( + """ + UPDATE crawl_target + SET mssp_route = @route, + mssp_route_at = CASE WHEN @route::text IS NULL THEN NULL ELSE @at END + WHERE id = @id + """, + new { id, route = MsspRouteSpelling.ToDb(route), at = at.ToUniversalTime() }, + cancellationToken: ct)); + } + /// /// Attaches the game this address turned out to be. /// @@ -244,6 +259,9 @@ private sealed class Row public string? MsspCharset { get; init; } + /// Text, not the enum, for the reason is. + public string? MsspRoute { get; init; } + /// /// Nullable because the subquery has no row to read for a target not yet bound to a game, /// and null there means the same as true: an address that has proved nothing yet is asked @@ -272,6 +290,35 @@ private sealed class Row Charset = Charset, MsspCharset = MsspCharset, AwaitingCorroboration = AwaitingCorroboration ?? true, + MsspRoute = MsspRouteSpelling.From(MsspRoute), }; } } + +/// +/// to and from the spellings +/// crawl_target_mssp_route_vocabulary allows. +/// +public static class MsspRouteSpelling +{ + public static string? ToDb(MsspRoute? route) => route switch + { + null => null, + MUI.Discovery.MsspRoute.Telnet => "telnet", + MUI.Discovery.MsspRoute.Plaintext => "plaintext", + MUI.Discovery.MsspRoute.None => "none", + _ => throw new ArgumentOutOfRangeException(nameof(route), route, "No spelling for this route."), + }; + + /// + /// An unknown spelling reads as unknown, which costs one trial dial rather than a crawl cycle + /// that throws on every row. + /// + public static MsspRoute? From(string? value) => value switch + { + "telnet" => MUI.Discovery.MsspRoute.Telnet, + "plaintext" => MUI.Discovery.MsspRoute.Plaintext, + "none" => MUI.Discovery.MsspRoute.None, + _ => null, + }; +} diff --git a/src/MUI.Discovery/Scheduling/CrawlTarget.cs b/src/MUI.Discovery/Scheduling/CrawlTarget.cs index 9eab3280..0ffeb4c3 100644 --- a/src/MUI.Discovery/Scheduling/CrawlTarget.cs +++ b/src/MUI.Discovery/Scheduling/CrawlTarget.cs @@ -129,6 +129,17 @@ public sealed record CrawlTarget /// /// public bool AwaitingCorroboration { get; init; } = true; + + /// + /// Which way this address hands over MSSP, as far as the crawl loop has learnt, or null when it + /// has learnt nothing yet (migration 0042). + /// + /// + /// The address's own column, like and for the same reason: a memory of what + /// a conversation with this listener established. owns both what it + /// asks for and how it changes. + /// + public MsspRoute? MsspRoute { get; init; } } /// @@ -172,5 +183,15 @@ Task RecordAttemptAsync( /// Task RecordTransportAsync(Guid id, bool useTls, CancellationToken ct); + /// + /// Records which way this address hands over MSSP, so the next dial knows whether to send the + /// plaintext MSSP-REQUEST. Null forgets it, which earns the address one more trial. + /// + /// + /// Separate from for the reason + /// is: most attempts have nothing new to say about it, and the caller writes only on a change. + /// + Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct); + Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct); } diff --git a/src/MUI.Discovery/Scheduling/MsspRoutes.cs b/src/MUI.Discovery/Scheduling/MsspRoutes.cs new file mode 100644 index 00000000..2356d64e --- /dev/null +++ b/src/MUI.Discovery/Scheduling/MsspRoutes.cs @@ -0,0 +1,93 @@ +using MUI.Crawl; + +namespace MUI.Discovery; + +/// +/// Which way an address hands over its MSSP report, as far as the crawl loop has learnt +/// (crawl_target.mssp_route, migration 0042). Null on a target means nothing is known yet. +/// +public enum MsspRoute +{ + /// A report has arrived over telnet option 70. The plaintext form is never sent. + Telnet, + + /// Answered the plaintext MSSP-REQUEST, so it is asked every session. + Plaintext, + + /// Sent MSSP-REQUEST and did not answer it. Never asked again. + None, +} + +/// +/// The two decisions exists for: what to ask this time, and what the answer +/// teaches. +/// +/// +/// Here rather than in the crawl loop so both are a pure function of a target and a +/// , testable without a socket or a database. +/// +public static class MsspRoutes +{ + /// What the probe is to do about MSSP-REQUEST at an address on this route. + /// + /// An unknown address gets one trial, on a dial of its own, and only once the probe has seen the + /// ordinary session come back without option 70 (). The probe + /// decides that last half, since only it has the session. + /// + public static PlaintextMsspAsk Ask(MsspRoute? route) => route switch + { + null => PlaintextMsspAsk.Trial, + MsspRoute.Plaintext => PlaintextMsspAsk.Ask, + _ => PlaintextMsspAsk.Never, + }; + + /// + /// The route this probe has taught, or when it taught nothing. + /// + /// + /// + /// Option 70 wins from any state, and once learnt is never unlearnt: the plaintext form is for + /// games that have not given us MSSP the proper way, and a game that has once is not asked it + /// again because one session's report went missing. Counted from the outcome rather than the + /// report, so a report too large to keep still counts: the game offered it, we declined it. + /// + /// + /// An unanswered request is final from the unknown state, which is the point. From + /// it returns the address to unknown instead, so the next crawl + /// gives it one more trial. A game that has answered before and misses once has more likely been + /// slow than changed its code, and losing its only MSSP route for ever on one slow night would be + /// our timing recorded as a fact about it (rule 5). A second miss, on the trial, settles it. + /// + /// + /// A failed dial teaches nothing, and neither does a session that never sent the line. + /// + /// + public static MsspRoute? Learn(MsspRoute? current, ProbeResult result) + { + ArgumentNullException.ThrowIfNull(result); + + if (result.Outcome is not ProbeOutcome.Answered) + { + return current; + } + + if (result.MsspOutcome is not MsspOutcome.NotOffered + && result.PlaintextMssp is not PlaintextMsspOutcome.Answered) + { + return MsspRoute.Telnet; + } + + if (current is MsspRoute.Telnet) + { + return current; + } + + return result.PlaintextMssp switch + { + PlaintextMsspOutcome.Answered => MsspRoute.Plaintext, + PlaintextMsspOutcome.Unanswered when current is MsspRoute.Plaintext => null, + PlaintextMsspOutcome.Unanswered => MsspRoute.None, + _ => current, + }; + } +} diff --git a/src/MUI.Probe/Program.cs b/src/MUI.Probe/Program.cs index dd36271e..079cce01 100644 --- a/src/MUI.Probe/Program.cs +++ b/src/MUI.Probe/Program.cs @@ -9,6 +9,14 @@ // catalogue to ask, and a caller that cannot answer keeps the probe asking. var listed = args.Contains("--listed", StringComparer.Ordinal); +// Whether to try the plaintext MSSP-REQUEST, as the crawl loop does for an address that has never +// reported over option 70: `--mssp-request` asks it on a second dial after the measurement +// (PlaintextMsspAsk.Trial), `--mssp-request=ask` inside the measurement, before WHO, as for an +// address known to answer. Off by default, because it is text at a stranger's login prompt. +var plaintextMssp = args.Contains("--mssp-request=ask", StringComparer.Ordinal) + ? PlaintextMsspAsk.Ask + : args.Contains("--mssp-request", StringComparer.Ordinal) ? PlaintextMsspAsk.Trial : PlaintextMsspAsk.Never; + // Flags are pulled out first so they can be written anywhere without displacing a positional. var positional = args.Where(a => !a.StartsWith("--", StringComparison.Ordinal)).ToArray(); @@ -35,6 +43,7 @@ Charset = charset, MsspCharset = msspCharset, AwaitingCorroboration = !listed, + PlaintextMssp = plaintextMssp, }); Console.WriteLine($"target {result.Host}:{result.Port}"); @@ -48,6 +57,11 @@ : $"transport none answered ({result.Transport.ToString().ToLowerInvariant()} attempted)"); Console.WriteLine($"elapsed {result.Elapsed.TotalSeconds:F1}s"); Console.WriteLine($"mssp {result.MsspOutcome} via {result.MsspTransport}"); + +if (plaintextMssp is not PlaintextMsspAsk.Never) +{ + Console.WriteLine($"mssp-request {result.PlaintextMssp}"); +} Console.WriteLine($"who {result.Who.Confidence}" + (result.Who.HasCount ? $" \u2192 {result.Who.Count} players" : result.Who.Attempted ? " \u2014 asked, unreadable" : " \u2014 never asked")); diff --git a/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs b/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs new file mode 100644 index 00000000..7f5c5de8 --- /dev/null +++ b/tests/MUI.Crawl.Tests/Telnet/PlaintextMsspTests.cs @@ -0,0 +1,339 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using MUI.Crawl; +using TelnetNegotiationCore.Builders; +using TelnetNegotiationCore.Interpreters; +using TelnetNegotiationCore.Models; +using TelnetNegotiationCore.Protocols; + +namespace MUI.Crawl.Tests; + +/// +/// The plaintext MSSP-REQUEST, end to end against servers in this process. +/// +/// +/// TelnetNegotiationCore owns the exchange itself and tests it there. What is pinned here is ours: +/// the line goes out only when the target says so, never to a game negotiating option 70, never +/// inside the session that measures a game for the first time, and what comes back is recorded as +/// MSSP without being recorded as a negotiation. +/// +public class PlaintextMsspTests +{ + private const string Request = "MSSP-REQUEST"; + + private static ProbeOptions Fast() => new() + { + QuietPeriod = TimeSpan.FromMilliseconds(120), + SilenceGrace = TimeSpan.FromMilliseconds(300), + MaxPhase = TimeSpan.FromSeconds(3), + BannerPatience = TimeSpan.FromMilliseconds(300), + WhoGrace = TimeSpan.FromMilliseconds(700), + PollInterval = TimeSpan.FromMilliseconds(15), + Timeout = TimeSpan.FromSeconds(20), + MsspSettleGrace = TimeSpan.FromMilliseconds(400), + PromptHold = TimeSpan.FromMilliseconds(120), + PlaintextMsspGrace = TimeSpan.FromMilliseconds(800), + }; + + [Test] + public async Task NothingIsSentUnlessTheTargetSaysSo() + { + await using var game = new OldRealms { AnswersRequest = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync(game.Target); + + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.NotAsked); + await Assert.That(game.Sessions).Count().IsEqualTo(1); + await Assert.That(game.Sessions.SelectMany(lines => lines)).DoesNotContain(Request); + } + + [Test] + public async Task ATrialAsksOnADialOfItsOwnAndKeepsTheReport() + { + await using var game = new OldRealms { AnswersRequest = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync( + game.Target with { PlaintextMssp = PlaintextMsspAsk.Trial }); + + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.Answered); + await Assert.That(result.MsspOutcome).IsEqualTo(MsspOutcome.Received); + await Assert.That(result.MsspTransport).IsEqualTo(MsspTransport.PlaintextRequest); + await Assert.That(MsspReport.Last(result.Mssp, "NAME")).IsEqualTo("Old Realms"); + await Assert.That(MsspReport.Last(result.Mssp, "PLAYERS")).IsEqualTo("4"); + + // A line of text is not a telnet negotiation, and the handshake field is a measurement of + // negotiations. Noting MSSP here would publish option 70 for a game that never offered it. + await Assert.That(result.OfferedOptions).DoesNotContain("MSSP"); + + // The measurement saw none of it, and the trial asked nothing else. + await Assert.That(game.Sessions).Count().IsEqualTo(2); + await Assert.That(game.Sessions[0]).DoesNotContain(Request); + await Assert.That(game.Sessions[0]).Contains("WHO"); + await Assert.That(game.Sessions[1]).IsEquivalentTo(new[] { Request }); + await Assert.That(result.Who.Count).IsEqualTo(3); + } + + /// + /// The reason the trial has a dial of its own. + /// + /// + /// The shape playdecay.com:3003 was measured in: an unknown name is answered with a + /// password prompt, and the next line is read as the password and ends the session. Had the + /// request gone out in the measuring session, WHO would have been typed as a password and + /// the hour recorded as uncountable — our experiment published as a fact about the game. + /// + [Test] + public async Task AGameThatReadsTheRequestAsANameHasItsMeasurementLeftAlone() + { + await using var game = new OldRealms { AsksForAPasswordAfterAName = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync( + game.Target with { PlaintextMssp = PlaintextMsspAsk.Trial }); + + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.Unanswered); + await Assert.That(result.MsspOutcome).IsEqualTo(MsspOutcome.NotOffered); + await Assert.That(result.MsspTransport).IsEqualTo(MsspTransport.None); + await Assert.That(result.Who.Count).IsEqualTo(3); + await Assert.That(game.Sessions[1]).IsEquivalentTo(new[] { Request }); + } + + [Test] + public async Task AGameThatHangsUpOnTheRequestHasAnsweredIt() + { + await using var game = new OldRealms { HangsUpOnAName = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync( + game.Target with { PlaintextMssp = PlaintextMsspAsk.Trial }); + + await Assert.That(result.Outcome).IsEqualTo(ProbeOutcome.Answered); + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.Unanswered); + await Assert.That(result.Who.Count).IsEqualTo(3); + } + + [Test] + public async Task AGameNegotiatingOption70IsNeverSentIt() + { + foreach (var ask in new[] { PlaintextMsspAsk.Trial, PlaintextMsspAsk.Ask }) + { + await using var game = new OldRealms { AnswersRequest = true, OffersOption70 = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync(game.Target with { PlaintextMssp = ask }); + + await Assert.That(result.MsspTransport).IsEqualTo(MsspTransport.TelnetOption70); + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.NotAsked); + await Assert.That(game.Sessions).Count().IsEqualTo(1); + await Assert.That(game.Sessions[0]).DoesNotContain(Request); + } + } + + [Test] + public async Task AKnownAnswererIsAskedBeforeWhoAndSparedTheWho() + { + await using var game = new OldRealms { AnswersRequest = true }; + + var result = await new TelnetProbe(Fast()).ProbeAsync( + game.Target with { PlaintextMssp = PlaintextMsspAsk.Ask }); + + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.Answered); + await Assert.That(result.MsspTransport).IsEqualTo(MsspTransport.PlaintextRequest); + await Assert.That(MsspReport.Last(result.Mssp, "PLAYERS")).IsEqualTo("4"); + await Assert.That(result.OfferedOptions).DoesNotContain("MSSP"); + + // One session: the request first, and WHO never, because the report stated a count. + await Assert.That(game.Sessions).Count().IsEqualTo(1); + await Assert.That(game.Sessions[0].First()).IsEqualTo(Request); + await Assert.That(game.Sessions[0]).DoesNotContain("WHO"); + + // The reply is protocol, not screen, and the prompt it re-printed is our doing. + await Assert.That(result.Banner).DoesNotContain("MSSP"); + await Assert.That(result.Banner).Contains("Welcome to Old Realms"); + } + + [Test] + public async Task AKnownAnswererThatFallsSilentIsRecordedAsSuch() + { + await using var game = new OldRealms(); + + var result = await new TelnetProbe(Fast()).ProbeAsync( + game.Target with { PlaintextMssp = PlaintextMsspAsk.Ask }); + + await Assert.That(result.PlaintextMssp).IsEqualTo(PlaintextMsspOutcome.Unanswered); + await Assert.That(result.MsspOutcome).IsEqualTo(MsspOutcome.NotOffered); + await Assert.That(game.Sessions).Count().IsEqualTo(1); + } + + /// + /// A connect screen at a name prompt, served to as many connections as arrive. + /// + /// + /// A TelnetNegotiationCore server for the framing, with the plaintext reply written by hand: + /// the library's own server half needs MSSPProtocol registered, which would offer option + /// 70 as well, and the game under test here is one that does not. The reply is written the way + /// SMAUG's send_mssp_data writes it. + /// + private sealed class OldRealms : IAsyncDisposable + { + private const string Prompt = "By what name do you wish to be known? "; + + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _stopping = new(); + private readonly List> _sessions = []; + private readonly Task _serving; + + public OldRealms() + { + _listener.Start(); + _serving = AcceptAsync(); + } + + public bool AnswersRequest { get; init; } + + public bool OffersOption70 { get; init; } + + /// Whether an unknown name is followed by a password prompt that ends the session. + public bool AsksForAPasswordAfterAName { get; init; } + + public bool HangsUpOnAName { get; init; } + + public ProbeTarget Target => new( + IPAddress.Loopback.ToString(), + ((IPEndPoint)_listener.LocalEndpoint).Port); + + public IReadOnlyList> Sessions + { + get + { + lock (_sessions) + { + return [.. _sessions.Select(lines => (IReadOnlyList)[.. lines])]; + } + } + } + + private async Task AcceptAsync() + { + var serving = new List(); + + try + { + while (!_stopping.IsCancellationRequested) + { + var client = await _listener.AcceptTcpClientAsync(_stopping.Token); + var lines = new List(); + + lock (_sessions) + { + _sessions.Add(lines); + } + + serving.Add(ServeAsync(client, lines)); + } + } + catch (Exception error) when (error is OperationCanceledException or SocketException) + { + } + + await Task.WhenAll(serving); + } + + private async Task ServeAsync(TcpClient client, List lines) + { + using var _ = client; + var awaitingPassword = false; + + try + { + var builder = new TelnetInterpreterBuilder() + .UseMode(TelnetInterpreter.TelnetMode.Server) + .UseLogger(NullLogger.Instance) + .OnSubmit(async (bytes, encoding, telnet) => + { + var line = encoding.GetString(bytes).Trim(); + + lock (_sessions) + { + lines.Add(line); + } + + if (awaitingPassword) + { + await WriteAsync(telnet, "Wrong password.\r\n"); + client.Client.Shutdown(SocketShutdown.Both); + return; + } + + switch (line.ToUpperInvariant()) + { + case "": + await WriteAsync(telnet, Prompt); + return; + + case Request when AnswersRequest: + await WriteAsync( + telnet, + "\r\nMSSP-REPLY-START\r\nNAME\tOld Realms\r\nPLAYERS\t4\r\n" + + "UPTIME\t1700000000\r\nMSSP-REPLY-END\r\n" + Prompt); + return; + + case "WHO": + await WriteAsync( + telnet, + "Player Name On For Idle\r\n" + + "3 Players logged in, 22 record, no maximum.\r\n" + Prompt); + return; + + case "INFO" or "VERSION": + await WriteAsync(telnet, Prompt); + return; + } + + if (HangsUpOnAName) + { + client.Client.Shutdown(SocketShutdown.Both); + } + else if (AsksForAPasswordAfterAName) + { + awaitingPassword = true; + await WriteAsync(telnet, "Password: "); + } + else + { + await WriteAsync(telnet, "Illegal name, try another.\r\n" + Prompt); + } + }); + + if (OffersOption70) + { + var mssp = new MSSPProtocol(); + mssp.SetMSSPConfig(() => new MSSPConfig { Name = "Old Realms", Players = 4 }); + builder = builder.AddPlugin(mssp); + } + + var built = await builder.BuildAndStartAsync(client, _stopping.Token); + await using var telnet = built.Interpreter; + + await WriteAsync(telnet, "Welcome to Old Realms\r\nA SMAUG-shaped place.\r\n" + Prompt); + await built.ReadTask; + } + catch (Exception error) when (error is OperationCanceledException + or IOException + or SocketException + or ObjectDisposedException) + { + } + } + + private static ValueTask WriteAsync(TelnetInterpreter telnet, string text) => + telnet.WriteToNetworkAsync(Encoding.Latin1.GetBytes(text)); + + public async ValueTask DisposeAsync() + { + await _stopping.CancelAsync(); + _listener.Stop(); + await _serving; + _stopping.Dispose(); + } + } +} diff --git a/tests/MUI.Crawl.Tests/Telnet/ProbeRestraintTests.cs b/tests/MUI.Crawl.Tests/Telnet/ProbeRestraintTests.cs index c9f467f2..2f38df31 100644 --- a/tests/MUI.Crawl.Tests/Telnet/ProbeRestraintTests.cs +++ b/tests/MUI.Crawl.Tests/Telnet/ProbeRestraintTests.cs @@ -40,11 +40,14 @@ public async Task TheOnlyThingsItAsksForAreThePreLoginReadOnlyCommands() } [Test] - public async Task PlaintextMsspRequestIsNeverTypedAtALoginScreen() + public async Task PlaintextMsspRequestIsNotACommandAndIsOffUnlessTheCrawlLoopAsks() { - // The plaintext MSSP-REQUEST form is text at a login screen, and most games tried read it as - // a character name. It belongs in TelnetNegotiationCore, not here. + // The plaintext MSSP-REQUEST form is text at a login screen, and eight of twenty games tried + // read it as a character name. TelnetNegotiationCore sends it, not this probe's command list, + // and only for a target the crawl loop has decided may be asked (PlaintextMsspTests). await Assert.That(TelnetProbe.PermittedCommands).DoesNotContain("MSSP-REQUEST"); + await Assert.That(new ProbeTarget("mud.example.org", 4000).PlaintextMssp) + .IsEqualTo(PlaintextMsspAsk.Never); } [Test] diff --git a/tests/MUI.Crawler.Tests/Crawl/MsspRoutePostgresTests.cs b/tests/MUI.Crawler.Tests/Crawl/MsspRoutePostgresTests.cs new file mode 100644 index 00000000..b65bdc46 --- /dev/null +++ b/tests/MUI.Crawler.Tests/Crawl/MsspRoutePostgresTests.cs @@ -0,0 +1,170 @@ +using Dapper; + +using MUI.Crawl; +using MUI.Crawler.Tests.Support; +using MUI.Discovery; + +namespace MUI.Crawler.Tests; + +/// +/// What the crawl loop remembers about the plaintext MSSP-REQUEST, and what it asks next time. +/// +/// +/// The probe decides how to send the line and reads what came back; these are about the memory +/// between dials (crawl_target.mssp_route), which is the whole of the "not again" in "try it, +/// and do not try it again on a game that does not answer". +/// +public class MsspRoutePostgresTests +{ + private const string Host = "realms.example.org"; + private const int Port = 4000; + + [Test] + public async Task AnAddressNobodyKnowsAnythingAboutIsGivenOneTrial() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + var probe = new ScriptedProbe(target => Probes.Answered(host: target.Host, port: target.Port)); + await CrawlCycles.Build(source, probe, TimeProvider.System).RunAsync(); + + await Assert.That(probe.Asked.Single().PlaintextMssp).IsEqualTo(PlaintextMsspAsk.Trial); + } + + [Test] + public async Task AGameThatAnsweredIsAskedInTheSessionFromThenOn() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + var trial = new ScriptedProbe(target => AnsweredInPlaintext(target)); + await CrawlCycles.Build(source, trial, TimeProvider.System).RunAsync(); + + await using var connection = await source.OpenConnectionAsync(); + + await Assert.That(await RouteAsync(connection)).IsEqualTo("plaintext"); + + // The report it sent is stored the way an option-70 report is: it is the same report. + await Assert.That(await connection.ExecuteScalarAsync( + "SELECT value FROM game_field WHERE field = 'NAME' AND source = 'mssp'")) + .IsEqualTo("Old Realms"); + + await DueAgainAsync(connection); + + var next = new ScriptedProbe(target => AnsweredInPlaintext(target)); + await CrawlCycles.Build(source, next, TimeProvider.System).RunAsync(); + + await Assert.That(next.Asked.Single().PlaintextMssp).IsEqualTo(PlaintextMsspAsk.Ask); + } + + [Test] + public async Task AGameThatDidNotAnswerIsNeverAskedAgain() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + var trial = new ScriptedProbe(target => Probes.Answered(host: target.Host, port: target.Port) with + { + PlaintextMssp = PlaintextMsspOutcome.Unanswered, + }); + await CrawlCycles.Build(source, trial, TimeProvider.System).RunAsync(); + + await using var connection = await source.OpenConnectionAsync(); + + await Assert.That(await RouteAsync(connection)).IsEqualTo("none"); + await Assert.That(await connection.ExecuteScalarAsync( + "SELECT mssp_route_at IS NOT NULL FROM crawl_target")).IsTrue(); + + await DueAgainAsync(connection); + + var next = new ScriptedProbe(target => Probes.Answered(host: target.Host, port: target.Port)); + await CrawlCycles.Build(source, next, TimeProvider.System).RunAsync(); + + await Assert.That(next.Asked.Single().PlaintextMssp).IsEqualTo(PlaintextMsspAsk.Never); + await Assert.That(await RouteAsync(connection)).IsEqualTo("none"); + } + + [Test] + public async Task AGameReportingOverOption70IsNeverSentTheRequest() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + var first = new ScriptedProbe(target => Probes.Answered( + host: target.Host, + port: target.Port, + mssp: Probes.Mssp(("NAME", "Old Realms")))); + await CrawlCycles.Build(source, first, TimeProvider.System).RunAsync(); + + await using var connection = await source.OpenConnectionAsync(); + + await Assert.That(await RouteAsync(connection)).IsEqualTo("telnet"); + + await DueAgainAsync(connection); + + // A session where the report went missing does not reopen the question. + var next = new ScriptedProbe(target => Probes.Answered(host: target.Host, port: target.Port)); + await CrawlCycles.Build(source, next, TimeProvider.System).RunAsync(); + + await Assert.That(next.Asked.Single().PlaintextMssp).IsEqualTo(PlaintextMsspAsk.Never); + await Assert.That(await RouteAsync(connection)).IsEqualTo("telnet"); + } + + [Test] + public async Task AFailedDialTeachesNothing() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + var probe = new ScriptedProbe(target => Probes.Failed(host: target.Host, port: target.Port)); + await CrawlCycles.Build(source, probe, TimeProvider.System).RunAsync(); + + await using var connection = await source.OpenConnectionAsync(); + + await Assert.That(await RouteAsync(connection)).IsNull(); + } + + [Test] + public async Task TheSchemaRefusesARouteNobodyDated() + { + await using var database = await PostgresFixture.MigratedAsync(); + var source = database.DataSource; + + await CrawlCycles.SeedAsync(source, new CrawlSeed(Host, Port)); + + await using var connection = await source.OpenConnectionAsync(); + + await Assert.That(async () => await connection.ExecuteAsync( + "UPDATE crawl_target SET mssp_route = 'none'")) + .Throws(); + await Assert.That(async () => await connection.ExecuteAsync( + "UPDATE crawl_target SET mssp_route = 'gopher', mssp_route_at = now()")) + .Throws(); + } + + private static ProbeResult AnsweredInPlaintext(ProbeTarget target) => + Probes.Answered( + host: target.Host, + port: target.Port, + mssp: Probes.Mssp(("NAME", "Old Realms"), ("PLAYERS", "4"))) with + { + MsspTransport = MsspTransport.PlaintextRequest, + PlaintextMssp = PlaintextMsspOutcome.Answered, + }; + + private static Task RouteAsync(Npgsql.NpgsqlConnection connection) => + connection.ExecuteScalarAsync("SELECT mssp_route FROM crawl_target"); + + private static Task DueAgainAsync(Npgsql.NpgsqlConnection connection) => + connection.ExecuteAsync("UPDATE crawl_target SET next_probe_at = now() - interval '1 hour'"); +} diff --git a/tests/MUI.Crawler.Tests/Support/FakeStores.cs b/tests/MUI.Crawler.Tests/Support/FakeStores.cs index f0c775f5..ec00405e 100644 --- a/tests/MUI.Crawler.Tests/Support/FakeStores.cs +++ b/tests/MUI.Crawler.Tests/Support/FakeStores.cs @@ -63,6 +63,9 @@ public Task RecordAttemptAsync( public Task RecordTransportAsync(Guid id, bool useTls, CancellationToken ct) => Task.CompletedTask; + public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) => + Task.CompletedTask; + public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) => Task.CompletedTask; } diff --git a/tests/MUI.Discovery.Tests/Scheduling/MsspRoutesTests.cs b/tests/MUI.Discovery.Tests/Scheduling/MsspRoutesTests.cs new file mode 100644 index 00000000..7925d315 --- /dev/null +++ b/tests/MUI.Discovery.Tests/Scheduling/MsspRoutesTests.cs @@ -0,0 +1,91 @@ +using MUI.Crawl; +using MUI.Discovery.Tests.Support; + +namespace MUI.Discovery.Tests; + +/// +/// When the plaintext MSSP-REQUEST is sent, and what each answer teaches. +/// +public class MsspRoutesTests +{ + private static ProbeResult Plaintext(PlaintextMsspOutcome outcome) => + ProbeResults.Answered() with { PlaintextMssp = outcome }; + + private static ProbeResult PlaintextReport() => + ProbeResults.Answered(mssp: ProbeResults.Mssp(("NAME", "Old Realms"))) with + { + MsspTransport = MsspTransport.PlaintextRequest, + PlaintextMssp = PlaintextMsspOutcome.Answered, + }; + + [Test] + public async Task OnlyAnUnknownAddressIsGivenATrialAndOnlyAKnownAnswererIsAsked() + { + await Assert.That(MsspRoutes.Ask(null)).IsEqualTo(PlaintextMsspAsk.Trial); + await Assert.That(MsspRoutes.Ask(MsspRoute.Plaintext)).IsEqualTo(PlaintextMsspAsk.Ask); + await Assert.That(MsspRoutes.Ask(MsspRoute.Telnet)).IsEqualTo(PlaintextMsspAsk.Never); + await Assert.That(MsspRoutes.Ask(MsspRoute.None)).IsEqualTo(PlaintextMsspAsk.Never); + } + + [Test] + public async Task AnOption70ReportSettlesItFromEveryState() + { + var report = ProbeResults.Answered(mssp: ProbeResults.Mssp(("NAME", "Old Realms"))); + + await Assert.That(MsspRoutes.Learn(null, report)).IsEqualTo(MsspRoute.Telnet); + await Assert.That(MsspRoutes.Learn(MsspRoute.None, report)).IsEqualTo(MsspRoute.Telnet); + await Assert.That(MsspRoutes.Learn(MsspRoute.Plaintext, report)).IsEqualTo(MsspRoute.Telnet); + } + + [Test] + public async Task AReportTooLargeToKeepIsStillAReportOverOption70() + { + var dropped = ProbeResults.Answered() with { MsspOutcome = MsspOutcome.RejectedTooLarge }; + + await Assert.That(MsspRoutes.Learn(null, dropped)).IsEqualTo(MsspRoute.Telnet); + } + + [Test] + public async Task OnceOption70IsKnownASessionWithoutItReopensNothing() + { + await Assert.That(MsspRoutes.Learn(MsspRoute.Telnet, ProbeResults.Answered())).IsEqualTo(MsspRoute.Telnet); + } + + [Test] + public async Task AnAnswerMakesItAPlaintextAddress() + { + await Assert.That(MsspRoutes.Learn(null, PlaintextReport())).IsEqualTo(MsspRoute.Plaintext); + await Assert.That(MsspRoutes.Learn(MsspRoute.Plaintext, PlaintextReport())).IsEqualTo(MsspRoute.Plaintext); + } + + [Test] + public async Task AnOversizedPlaintextAnswerIsStillAnAnswer() + { + var dropped = Plaintext(PlaintextMsspOutcome.Answered) with { MsspOutcome = MsspOutcome.RejectedTooLarge }; + + await Assert.That(MsspRoutes.Learn(null, dropped)).IsEqualTo(MsspRoute.Plaintext); + } + + [Test] + public async Task NoAnswerToTheTrialIsFinal() + { + await Assert.That(MsspRoutes.Learn(null, Plaintext(PlaintextMsspOutcome.Unanswered))) + .IsEqualTo(MsspRoute.None); + } + + [Test] + public async Task AFormerAnswererThatMissesOnceGetsOneMoreTrialRatherThanBeingDropped() + { + await Assert.That(MsspRoutes.Learn(MsspRoute.Plaintext, Plaintext(PlaintextMsspOutcome.Unanswered))) + .IsNull(); + } + + [Test] + public async Task NotAskingAndFailingToDialTeachNothing() + { + await Assert.That(MsspRoutes.Learn(null, ProbeResults.Answered())).IsNull(); + await Assert.That(MsspRoutes.Learn(MsspRoute.Plaintext, ProbeResults.Answered())).IsEqualTo(MsspRoute.Plaintext); + await Assert.That(MsspRoutes.Learn(MsspRoute.None, ProbeResults.Failed())).IsEqualTo(MsspRoute.None); + await Assert.That(MsspRoutes.Learn(null, ProbeResults.Failed())).IsNull(); + } +} diff --git a/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs b/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs index c7701d38..dd9557ff 100644 --- a/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs +++ b/tests/MUI.Discovery.Tests/Support/InMemoryCrawlTargetRepository.cs @@ -71,6 +71,13 @@ public Task RecordTransportAsync(Guid id, bool useTls, CancellationToken ct) return Task.CompletedTask; } + public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) + { + Replace(id, target => target with { MsspRoute = route }); + + return Task.CompletedTask; + } + public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) { Replace(id, target => target.GameId is null ? target with { GameId = gameId } : target); diff --git a/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs b/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs index ad5f7bd9..c3382bba 100644 --- a/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs +++ b/tests/MUI.Web.Tests/StoredCrawlerPulseTests.cs @@ -180,6 +180,9 @@ public Task RecordAttemptAsync( public Task RecordTransportAsync(Guid id, bool useTls, CancellationToken ct) => throw new NotSupportedException(); + public Task RecordMsspRouteAsync(Guid id, MsspRoute? route, DateTimeOffset at, CancellationToken ct) => + throw new NotSupportedException(); + public Task AttachGameAsync(Guid id, Guid gameId, CancellationToken ct) => throw new NotSupportedException(); }