From e7b9e83dd3d968706025d2ac554d3a1098ebe814 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Wed, 7 Oct 2026 16:15:01 +0200 Subject: [PATCH 1/7] Retire the shell when a command's terminate Signal does not go through A long-lived client polling periodically ended up with every command refused with WSManFault 2150859174 (maximum number of concurrent operations for this user exceeded), until the client was recreated. When the terminate Signal ending a command faulted, failed in transit, or was skipped for lack of budget in a bounded poll, the client kept reusing the shell, and the never-terminated command kept holding one of the user's WSMan operations until the shell was deleted, which only close() did. The same happened when the Signal stopping a still-running command failed. Such a shell is now retired: the next command deletes it and runs in a fresh shell created with the same working directory, environment and profile, and close() deletes it when no command follows. The Delete is deferred rather than sent right away because a completed command's cleanup must neither outlive the caller's poll budget nor fail it, and the failed or skipped Signal leaves no budget, and possibly no connection, for it. The command that follows restores its own timeouts after the Delete (a timed-out Delete must not leave the streaming per-leg deadline expired for the Create's reconnect) and checks for cancellation before creating the new shell. Closes #196 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++ .../metricshub/winrm/light/WsmanClient.java | 70 +++++++++++++- src/site/markdown/commands.md | 3 +- src/site/markdown/preparing-the-host.md | 2 +- .../metricshub/winrm/StreamingApiTest.java | 77 +++++++++++++++ .../winrm/light/WsmanProtocolTest.java | 96 +++++++++++++++++++ 6 files changed, 250 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5631deb..e9a3c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,6 +158,15 @@ Consequences: ### Fixed +- **A long-lived client no longer exhausts the host's operation quota** (#196). When the + terminate `Signal` ending a command failed (a fault, a dropped connection) or was skipped (no + time left in a bounded poll), the client kept reusing the shell, and the command, never + terminated, kept holding one of the user's WSMan operations. A client polling indefinitely + piled them up until every command was refused with WSManFault 2150859174 (*the maximum number + of concurrent operations for this user has been exceeded*), until the client was recreated. + Such a shell is now retired: the next command deletes it and runs in a fresh shell, created + with the same working directory, environment and profile. + - **WQL array properties keep all their elements** (#189). A WMI array (`IPAddress`, `DefaultIPGateway`, `Capabilities`, ...) used to yield only its last element, without any error. Its elements are now joined with `|` (`"192.0.2.10|fe80::1"`); the new diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index 73ed7e5..784c32d 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -107,6 +107,14 @@ final class WsmanClient implements AutoCloseable { private String pendingAuthorization; private String shellId; + // A shell no longer reused because one of its commands may never have been terminated: its + // terminate Signal faulted, failed in transit, or found no budget (issue #196). Such a command + // keeps holding one of the user's WSMan operations until its shell is deleted, so reusing the + // shell would pile them up until MaxConcurrentOperationsPerUser refuses every command. Deleted + // before the next shell is created, or by close(); never set together with shellId. Guarded by + // connectionPermit, like shellId. + private String retiredShellId; + // The shell's working directory and environment variables are pinned by the FIRST command on // this connection and reused whenever the shell must be (re)created — e.g. after the server // reaped it — so a recreation stays invisible to the caller instead of silently moving later @@ -560,6 +568,14 @@ RemoteCommand startCommand( shellSettingsPinned = true; } if (shellId == null) { + if (retiredShellId != null) { + deleteRetiredShell(operationTimeoutMs); + // The Delete was a round trip of its own: in streaming mode a timed-out one leaves + // the transport's per-leg deadline expired, which would give the Create's reconnect + // a 1 ms budget. And never create a shell after the caller's timeout was reported. + configureTimeouts(operationTimeoutMs, failOnQuietTimeout); + checkNotCancelled(); + } createShell(shellWorkingDirectory, shellEnvironment, operationTimeoutMs, failOnQuietTimeout); } // The caller's timeout may have fired while the Create response was being awaited (socket @@ -925,15 +941,20 @@ private void finish() throws Exception { * Terminate a command closed before it completed: the Signal is what stops it, so its * failures are reported — except the expiry of its short hold (see * {@link #EARLY_CLOSE_SIGNAL_MS}), a complete exchange that leaves the connection in sync - * and the command killed. + * and the command killed. A reported failure also retires the shell (see + * {@link #retireShell()}): the command may still be running in it. */ private void terminateRunning() throws Exception { try { terminate(commandId, Math.min(EARLY_CLOSE_SIGNAL_MS, operationTimeoutMs)); } catch (final WinRMFaultException e) { if (!FAULT_OPERATION_TIMEOUT.equals(e.getFaultCode())) { + retireShell(); throw e; } + } catch (final Exception e) { + retireShell(); + throw e; } } @@ -962,23 +983,27 @@ private void finishBounded(final long budgetMs) { * complete, in-sync exchange and is simply ignored; any other failure (a timeout, a reset, * a half-read response) leaves the connection in an unknown state, so it is dropped — a * late response must not desync a later request. A budget too small for any round trip - * skips the Signal outright, leaving the healthy connection untouched; the server reaps - * the completed command's state with the shell. + * skips the Signal outright, leaving the healthy connection untouched. Whenever the Signal + * does not go through, the command keeps holding a WSMan operation until its shell is + * deleted, so the shell is retired (see {@link #retireShell()}) — never reused. */ private void terminateCompleted(final long budgetMs) { // Same strict clamp as the bounded poll: the per-round-trip timeout caps this cleanup too. final long budget = Math.max(1, Math.min(budgetMs, operationTimeoutMs)); if (budget < MIN_WIRE_POLL_MS) { + retireShell(); return; } transport.pollTimeout(toSocketTimeoutMillis(budget)); try { terminate(commandId, budget); - } catch (final WinRMFaultException ignored) { + } catch (final WinRMFaultException e) { // The Signal was answered with a fault: the exchange completed, the connection is in // sync — and the command's completion is what matters. + retireShell(); } catch (final Exception e) { transport.close(); + retireShell(); } finally { transport.inactivityTimeout(toSocketTimeoutMillis(operationTimeoutMs)); } @@ -1059,6 +1084,39 @@ private void signal(final String commandId, final String code, final long timeou } } + /** + * Stop reusing the current shell: one of its commands may never have been terminated, and only + * deleting the shell releases the WSMan operation that command holds (issue #196). The Delete + * is deferred to the next shell creation or to {@link #close()}: the failed or skipped Signal + * that got us here leaves no budget, and possibly no connection, to send it now. + */ + private void retireShell() { + if (shellId != null) { + retiredShellId = shellId; + shellId = null; + } + } + + /** + * Best-effort Delete of the retired shell, sent right before its replacement is created. No + * failure of it may fail the new command, which owes nothing to its predecessor's cleanup: a + * shell this Delete cannot reach is left to the server's IdleTimeout, and a broken round trip + * has already dropped the connection (see {@link HttpTransport#post}), so the Create that + * follows starts on a fresh one. + */ + private void deleteRetiredShell(final long timeoutMs) { + final String shell = retiredShellId; + retiredShellId = null; + try { + request(Envelopes.deleteShell(url, shell, timeoutMs)); + } catch (final InterruptedException e) { + // Keep the cancellation visible to the checks that guard the next steps + Thread.currentThread().interrupt(); + } catch (final Exception ignored) { + // best-effort shell cleanup + } + } + // --- transport / crypto ------------------------------------------------- /** Send a request, expecting HTTP 200; throw with the WSMan fault detail otherwise. */ @@ -1452,8 +1510,10 @@ public void close() { // transport — which unblocks that worker's read; the shell is reaped by the server IdleTimeout. final boolean locked = connectionPermit.tryAcquire(); try { - final String shell = shellId; + // At most one of the two is set: a retired shell is deleted before the next one is created. + final String shell = shellId != null ? shellId : retiredShellId; shellId = null; + retiredShellId = null; if (locked && shell != null) { try { send(Envelopes.deleteShell(url, shell, timeoutMs)); diff --git a/src/site/markdown/commands.md b/src/site/markdown/commands.md index 0bb624a..2d4dd62 100644 --- a/src/site/markdown/commands.md +++ b/src/site/markdown/commands.md @@ -75,7 +75,8 @@ try (WinRMClient client = WinRMClient.builder("server.example.com") The profile is loaded when the remote shell is created, so this is a client setting. It applies to every shell the client creates, for commands, file transfers and remote file operations alike, -including a shell recreated after the server reaped the previous one. Microsoft's `winrs` +including a shell recreated after the server reaped the previous one, or after the client deleted +one holding a command it could not terminate. Microsoft's `winrs` documentation warns that loading the profile fails for a user who is not a local administrator on the host: the command then fails with a [`WinRMFaultException`](apidocs/org/metricshub/winrm/exceptions/WinRMFaultException.html) carrying diff --git a/src/site/markdown/preparing-the-host.md b/src/site/markdown/preparing-the-host.md index b56afac..a3a5fed 100644 --- a/src/site/markdown/preparing-the-host.md +++ b/src/site/markdown/preparing-the-host.md @@ -369,7 +369,7 @@ the host, the tighter they are: | --- | --- | --- | | `MaxMemoryPerShellMB` | Memory per shell, including child processes | Historically **150 MB**; 1024 MB on modern hosts. A command whose output is large can hit it. | | `MaxShellsPerUser` | Concurrent shells per user | 5 on older hosts, 30 on modern ones. Close clients you no longer need. | -| `MaxConcurrentOperationsPerUser` | Concurrent operations per user | 15 on Windows Server 2008 R2, 1500 later. File transfers are batched specifically to stay under low limits. | +| `MaxConcurrentOperationsPerUser` | Concurrent operations per user | 15 on Windows Server 2008 R2, 1500 later. File transfers are batched specifically to stay under low limits. A command holds one until it is terminated; when the client cannot terminate it, it deletes that shell rather than reuse it, so a long-lived client does not pile them up. | | `MaxEnvelopeSizekb` | SOAP envelope size | 150 KB on older hosts, **500 KB** on modern ones. This client always uses 150 KB envelopes, so the default never needs raising. | | `IdleTimeout` | How long an idle shell survives | 180000 ms (3 min) on older hosts, **7200000 ms** (2 h) on modern ones; 60000 ms minimum. The client transparently recreates a shell the host reaped. | diff --git a/src/test/java/org/metricshub/winrm/StreamingApiTest.java b/src/test/java/org/metricshub/winrm/StreamingApiTest.java index 831a360..f7d9fef 100644 --- a/src/test/java/org/metricshub/winrm/StreamingApiTest.java +++ b/src/test/java/org/metricshub/winrm/StreamingApiTest.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.metricshub.winrm.light.FakeWsmanResponses.commandResponse; import static org.metricshub.winrm.light.FakeWsmanResponses.done; +import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueShellDeletion; import static org.metricshub.winrm.light.FakeWsmanResponses.envelope; import static org.metricshub.winrm.light.FakeWsmanResponses.enumerationDone; import static org.metricshub.winrm.light.FakeWsmanResponses.fault; @@ -784,6 +785,82 @@ void completionInsideATinyPollSkipsTheSignal() throws Exception { } } + @Test + void aSkippedCompletionSignalRetiresTheShell() throws Exception { + enqueueCommandStartup(); + server.enqueue(200, envelope(receiveResponse(stdoutChunk("done\n"), done(COMMAND_ID, 5)))); + enqueueNextCommandInANewShell(); + + try (WinRMClient client = builder().build()) { + try (CommandCursor cursor = client.executor().startCommand("run.exe", null, 10_000)) { + cursor.poll(5_000); + // No round trip fits the budget: the Signal is skipped, the command never terminated. + assertNull(cursor.poll(20)); + } + // Issue #196: reusing SHELL-1 would leave that command holding a WSMan operation for good. + assertNextCommandRunsInANewShell(client); + } + } + + @Test + void aFailedEarlyCloseSignalRetiresTheShell() throws Exception { + enqueueCommandStartup(); + // The Signal stopping the still-running command is rejected: the failure is reported, and + // the command may still be running in SHELL-1. + server.enqueue(500, fault("999", "Signal rejected")); + enqueueNextCommandInANewShell(); + + try (WinRMClient client = builder().build()) { + final CommandCursor cursor = client.executor().startCommand("run.exe", null, 10_000); + assertThrows(WinRMClientException.class, cursor::close); + assertNextCommandRunsInANewShell(client); + } + } + + @Test + void anEarlyCloseSignalLostInTransitRetiresTheShell() throws Exception { + enqueueCommandStartup(); + // The connection drops before the Signal is answered: whether the command was stopped is + // unknown, so SHELL-1 must not be reused either. + server.enqueueDrop(); + enqueueNextCommandInANewShell(); + + try (WinRMClient client = builder().build()) { + final CommandCursor cursor = client.executor().startCommand("run.exe", null, 10_000); + assertThrows(WinRMClientException.class, cursor::close); + assertNextCommandRunsInANewShell(client); + } + } + + /** Script the next command: the retired shell's Delete, then a whole command in SHELL-2. */ + private void enqueueNextCommandInANewShell() { + enqueueShellDeletion(server); + server + .enqueue(200, envelope(resourceCreated("SHELL-2"))) + .enqueue(200, envelope(commandResponse("CMD-2"))) + .enqueue( + 200, + envelope(receiveResponse(stream("stdout", "CMD-2", "two".getBytes(StandardCharsets.UTF_8)), done("CMD-2", 0))) + ) + .enqueue(200, envelope(signalResponse())); + } + + /** Run the next command: it must delete SHELL-1 and run in a fresh SHELL-2, never reuse SHELL-1. */ + private void assertNextCommandRunsInANewShell(final WinRMClient client) throws Exception { + final int first = server.decryptedRequests().size(); + try (CommandCursor cursor = client.executor().startCommand("next.exe", null, 10_000)) { + assertEquals("two", new String(cursor.next().stdout(), StandardCharsets.UTF_8)); + assertNull(cursor.next()); + } + final List requests = server.decryptedRequests(); + final String delete = requests.get(first); + assertTrue(delete.contains("transfer/Delete"), delete); + assertTrue(delete.contains("Selector Name=\"ShellId\">SHELL-1<"), delete); + assertTrue(requests.get(first + 1).contains("transfer/Create"), requests.get(first + 1)); + final String command = requests.get(first + 2); + assertTrue(command.contains("Selector Name=\"ShellId\">SHELL-2<"), command); + } + @Test void commandSilenceBeyondTheTimeoutSurfacesAsInactivityTimeout() throws Exception { enqueueCommandStartup(); diff --git a/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java b/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java index 1ffc664..f401e5f 100644 --- a/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java +++ b/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java @@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.metricshub.winrm.light.FakeWsmanResponses.commandResponse; import static org.metricshub.winrm.light.FakeWsmanResponses.done; +import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueShellDeletion; import static org.metricshub.winrm.light.FakeWsmanResponses.envelope; import static org.metricshub.winrm.light.FakeWsmanResponses.fault; import static org.metricshub.winrm.light.FakeWsmanResponses.instance; @@ -408,6 +409,101 @@ void terminateSignalToleratesShellNotFoundFault() throws Exception { } } + @Test + void aFaultedTerminateSignalRetiresTheShell() throws Exception { + // Issue #196: a command whose terminate Signal failed keeps holding one of the user's WSMan + // operations until its shell is deleted. A long-lived client that kept reusing the shell piled + // them up until MaxConcurrentOperationsPerUser refused every Command. + server + .enqueue(200, envelope(resourceCreated("SHELL-1"))) + .enqueue(200, envelope(commandResponse("CMD-1"))) + .enqueue( + 200, + envelope(receiveResponse(stream("stdout", "CMD-1", "one".getBytes(StandardCharsets.UTF_8)), done("CMD-1", 0))) + ) + .enqueue(500, fault("999", "Signal rejected")); + enqueueNextCommandInANewShell(); + + assertNextCommandRunsInANewShell(); + } + + @Test + void aTerminateSignalLostInTransitRetiresTheShell() throws Exception { + // Same as a faulted Signal, but the connection drops before the answer: whether the command + // was terminated is unknown, so its shell must not be reused either. + server + .enqueue(200, envelope(resourceCreated("SHELL-1"))) + .enqueue(200, envelope(commandResponse("CMD-1"))) + .enqueue( + 200, + envelope(receiveResponse(stream("stdout", "CMD-1", "one".getBytes(StandardCharsets.UTF_8)), done("CMD-1", 0))) + ) + .enqueueDrop(); + enqueueNextCommandInANewShell(); + + assertNextCommandRunsInANewShell(); + } + + @Test + void closeDeletesARetiredShell() throws Exception { + // The last command's Signal failed, and no other command follows to delete its shell: close() + // must. A client per collection cycle would otherwise leave a shell and its held operation behind + // every cycle. + server + .enqueue(200, envelope(resourceCreated("SHELL-1"))) + .enqueue(200, envelope(commandResponse("CMD-1"))) + .enqueue( + 200, + envelope(receiveResponse(stream("stdout", "CMD-1", "one".getBytes(StandardCharsets.UTF_8)), done("CMD-1", 0))) + ) + .enqueue(500, fault("999", "Signal rejected")); + enqueueShellDeletion(server); + + try (LightWinRMService service = client(PASSWORD)) { + assertEquals("one", service.executeCommand("poll", null, StandardCharsets.UTF_8, TIMEOUT).getStdout()); + } + + final List requests = server.decryptedRequests(); + final String delete = requests.get(requests.size() - 1); + assertTrue(delete.contains("transfer/Delete"), delete); + assertTrue(delete.contains("Selector Name=\"ShellId\">SHELL-1<"), delete); + } + + /** Script the second command: the retired shell's Delete, then a whole command in SHELL-2. */ + private void enqueueNextCommandInANewShell() { + enqueueShellDeletion(server); + server + .enqueue(200, envelope(resourceCreated("SHELL-2"))) + .enqueue(200, envelope(commandResponse("CMD-2"))) + .enqueue( + 200, + envelope(receiveResponse(stream("stdout", "CMD-2", "two".getBytes(StandardCharsets.UTF_8)), done("CMD-2", 0))) + ) + .enqueue(200, envelope(signalResponse())); + } + + /** + * Run two commands on one client, the first one's terminate Signal failing: both must succeed, + * and the second must delete SHELL-1 and run in a fresh SHELL-2 instead of reusing SHELL-1. + */ + private void assertNextCommandRunsInANewShell() throws Exception { + try (LightWinRMService service = client(PASSWORD)) { + // The failed Signal is cleanup noise: the completed command's result is still reported. + assertEquals("one", service.executeCommand("poll", null, StandardCharsets.UTF_8, TIMEOUT).getStdout()); + assertEquals("two", service.executeCommand("poll", null, StandardCharsets.UTF_8, TIMEOUT).getStdout()); + } + + final List requests = server.decryptedRequests(); + // Create, Command, Receive, the failed Signal, then the second command's requests. + assertTrue(requests.size() >= 9, () -> String.join("\n---\n", requests)); + final String delete = requests.get(4); + assertTrue(delete.contains("transfer/Delete"), delete); + assertTrue(delete.contains("Selector Name=\"ShellId\">SHELL-1<"), delete); + assertTrue(requests.get(5).contains("transfer/Create"), requests.get(5)); + final String command = requests.get(6); + assertTrue(command.contains("Selector Name=\"ShellId\">SHELL-2<"), command); + } + // --- Fault mapping ------------------------------------------------------------ @Test From 7664221edbb9418ce5881dc798b24da7446d5f47 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Wed, 7 Oct 2026 16:25:17 +0200 Subject: [PATCH 2/7] Drop the connection when the retired shell's Delete fails Address Codex review: a Delete answered with an unexpected HTTP status (e.g. 503) is rejected by send() before its sealed body is decrypted, while HttpTransport.post() keeps the socket open because the exchange completed. Swallowing that failure let the Create that follows reuse an NTLM session whose incoming cipher stream was out of sync, failing the new command with a checksum mismatch. Any failure of the best-effort Delete other than a WSMan fault now closes the transport, so the Create starts on a fresh, re-authenticated connection, as terminateCompleted() already does for the Signal. Co-Authored-By: Claude Opus 5.5 --- .../metricshub/winrm/light/WsmanClient.java | 12 ++++++---- .../winrm/light/WsmanProtocolTest.java | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index 784c32d..994769c 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -1100,9 +1100,10 @@ private void retireShell() { /** * Best-effort Delete of the retired shell, sent right before its replacement is created. No * failure of it may fail the new command, which owes nothing to its predecessor's cleanup: a - * shell this Delete cannot reach is left to the server's IdleTimeout, and a broken round trip - * has already dropped the connection (see {@link HttpTransport#post}), so the Create that - * follows starts on a fresh one. + * shell this Delete cannot reach is left to the server's IdleTimeout. A failure other than a + * WSMan fault drops the connection, so the Create that follows starts on a fresh, + * re-authenticated one: a response rejected for its HTTP status is never decrypted, which + * leaves the message encryption out of sync on this connection. */ private void deleteRetiredShell(final long timeoutMs) { final String shell = retiredShellId; @@ -1112,8 +1113,9 @@ private void deleteRetiredShell(final long timeoutMs) { } catch (final InterruptedException e) { // Keep the cancellation visible to the checks that guard the next steps Thread.currentThread().interrupt(); - } catch (final Exception ignored) { - // best-effort shell cleanup + } catch (final Exception e) { + // best-effort shell cleanup, on a connection whose state is now unknown + transport.close(); } } diff --git a/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java b/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java index f401e5f..5237146 100644 --- a/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java +++ b/src/test/java/org/metricshub/winrm/light/WsmanProtocolTest.java @@ -62,6 +62,7 @@ class WsmanProtocolTest { private static final String WSEN = "http://schemas.xmlsoap.org/ws/2004/09/enumeration"; private static final String WSMAN = "http://schemas.dmtf.org/wbem/wsman/1/wsman.xsd"; private static final String RSP = "http://schemas.microsoft.com/wbem/wsman/1/windows/shell"; + private static final String DELETE_RESPONSE = ""; private FakeWsmanServer server; @@ -469,9 +470,32 @@ void closeDeletesARetiredShell() throws Exception { assertTrue(delete.contains("Selector Name=\"ShellId\">SHELL-1<"), delete); } + @Test + void aRetiredShellDeleteAnsweredWithAnUnexpectedStatusDoesNotFailTheNextCommand() throws Exception { + // The best-effort Delete draws an HTTP 503 whose sealed body is never decrypted: reusing that + // connection would decrypt the Create's response with an out-of-sync cipher stream. + server + .enqueue(200, envelope(resourceCreated("SHELL-1"))) + .enqueue(200, envelope(commandResponse("CMD-1"))) + .enqueue( + 200, + envelope(receiveResponse(stream("stdout", "CMD-1", "one".getBytes(StandardCharsets.UTF_8)), done("CMD-1", 0))) + ) + .enqueue(500, fault("999", "Signal rejected")) + .enqueue(503, envelope(DELETE_RESPONSE)); + enqueueCommandInShell2(); + + assertNextCommandRunsInANewShell(); + } + /** Script the second command: the retired shell's Delete, then a whole command in SHELL-2. */ private void enqueueNextCommandInANewShell() { enqueueShellDeletion(server); + enqueueCommandInShell2(); + } + + /** Script a whole command in a freshly created SHELL-2. */ + private void enqueueCommandInShell2() { server .enqueue(200, envelope(resourceCreated("SHELL-2"))) .enqueue(200, envelope(commandResponse("CMD-2"))) From d9852326851c75742a556a6a1f0fc48361f43420 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Wed, 7 Oct 2026 16:35:40 +0200 Subject: [PATCH 3/7] Keep the shell retired when its Delete is cancelled before being sent Address Codex review: with connection retries enabled, the deferred Delete of a retired shell can be cancelled by the command's wall-clock deadline during a connect-retry pause, before the request is sent. The retired shell ID was already cleared, so a later command created a new shell without retrying the cleanup, leaving the old command's operation allocated until the server's IdleTimeout. The ID is now restored on that cancellation path, so the next command or close() still deletes it; the restored interrupt makes the cancelled command abort before its Create, so the shell is never retired alongside a new one. Co-Authored-By: Claude Opus 5.5 --- .../metricshub/winrm/light/WsmanClient.java | 5 +- .../winrm/light/WsmanRetryTest.java | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index 994769c..0c71209 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -1111,7 +1111,10 @@ private void deleteRetiredShell(final long timeoutMs) { try { request(Envelopes.deleteShell(url, shell, timeoutMs)); } catch (final InterruptedException e) { - // Keep the cancellation visible to the checks that guard the next steps + // Cancelled in a connect-retry pause, before the Delete was sent: keep the shell retired so + // a later command or close() still deletes it. The restored interrupt makes the caller + // abort before its Create, so the shell is never retired alongside a new one. + retiredShellId = shell; Thread.currentThread().interrupt(); } catch (final Exception e) { // best-effort shell cleanup, on a connection whose state is now unknown diff --git a/src/test/java/org/metricshub/winrm/light/WsmanRetryTest.java b/src/test/java/org/metricshub/winrm/light/WsmanRetryTest.java index 372892b..135ae91 100644 --- a/src/test/java/org/metricshub/winrm/light/WsmanRetryTest.java +++ b/src/test/java/org/metricshub/winrm/light/WsmanRetryTest.java @@ -23,9 +23,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.metricshub.winrm.light.FakeWsmanResponses.COMMAND_ID; +import static org.metricshub.winrm.light.FakeWsmanResponses.commandResponse; +import static org.metricshub.winrm.light.FakeWsmanResponses.done; +import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueCommandExchange; import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueEnumeration; import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueShellCreation; +import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueShellDeletion; +import static org.metricshub.winrm.light.FakeWsmanResponses.envelope; import static org.metricshub.winrm.light.FakeWsmanResponses.instance; +import static org.metricshub.winrm.light.FakeWsmanResponses.receiveResponse; +import static org.metricshub.winrm.light.FakeWsmanResponses.resourceCreated; +import static org.metricshub.winrm.light.FakeWsmanResponses.stream; import java.net.ServerSocket; import java.nio.charset.StandardCharsets; @@ -263,6 +272,43 @@ void closeDuringARetryPauseAbortsInsteadOfRevivingTheConnection() throws Excepti } } + @Test + void aRetiredShellDeleteCancelledInARetryPauseIsSentByTheNextCommand() throws Exception { + // The first command's terminate Signal is lost in transit: SHELL-1 is retired (issue #196). + enqueueShellCreation(server); + server + .enqueue(200, envelope(commandResponse(COMMAND_ID))) + .enqueue( + 200, + envelope( + receiveResponse(stream("stdout", COMMAND_ID, "one".getBytes(StandardCharsets.UTF_8)), done(COMMAND_ID, 0)) + ) + ) + .enqueueDrop(); + + try (LightWinRMService service = service(server.port(), 3, 5_000L)) { + assertEquals("one", service.executeCommand("poll", null, StandardCharsets.UTF_8, TIMEOUT).getStdout()); + + // The second command's Delete must reconnect: the handshake is dropped, and the deadline + // cancels the command during the retry pause, before the Delete was ever sent. + server.dropNextConnections(1); + assertThrows(TimeoutException.class, () -> service.executeCommand("poll", null, StandardCharsets.UTF_8, 1_500L)); + + // SHELL-1 is still retired: the third command deletes it before creating SHELL-2. + enqueueShellDeletion(server); + server.enqueue(200, envelope(resourceCreated("SHELL-2"))); + enqueueCommandExchange(server, "two".getBytes(StandardCharsets.UTF_8), new byte[0], 0); + assertEquals("two", service.executeCommand("poll", null, StandardCharsets.UTF_8, TIMEOUT).getStdout()); + } + + final List requests = server.decryptedRequests(); + // Create, Command, Receive, the dropped Signal, then the third command's requests. + final String delete = requests.get(4); + assertTrue(delete.contains("transfer/Delete"), delete); + assertTrue(delete.contains("Selector Name=\"ShellId\">SHELL-1<"), delete); + assertTrue(requests.get(6).contains("Selector Name=\"ShellId\">SHELL-2<"), requests.get(6)); + } + // --- the wall-clock deadline still governs --------------------------------- @Test From d257bdf828329bdcbaa1c5ccdc7e5cc44dcdc668 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Wed, 7 Oct 2026 19:15:54 +0200 Subject: [PATCH 4/7] Retire the shell when an early-close Signal outlasts its hold Address review: the OperationTimeout fault answering the terminate Signal of a command closed before completion only says the service did not finish processing the Signal within its 1 s hold, not that the command is gone. Keeping the shell bet on the command having been killed; if it was not, its operation stayed allocated, which is #196 on this path. The shell is now retired on that fault too: close() still succeeds, and the next command deletes the shell and creates a new one. Co-Authored-By: Claude Opus 5.5 --- .../org/metricshub/winrm/light/WsmanClient.java | 7 ++++--- .../org/metricshub/winrm/StreamingApiTest.java | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index 0c71209..517a578 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -941,15 +941,16 @@ private void finish() throws Exception { * Terminate a command closed before it completed: the Signal is what stops it, so its * failures are reported — except the expiry of its short hold (see * {@link #EARLY_CLOSE_SIGNAL_MS}), a complete exchange that leaves the connection in sync - * and the command killed. A reported failure also retires the shell (see - * {@link #retireShell()}): the command may still be running in it. + * and the shell retired. Any failure, that expiry included, retires the shell (see + * {@link #retireShell()}): the fault only says the service did not finish processing the + * Signal in time, so the command may still be running in it. */ private void terminateRunning() throws Exception { try { terminate(commandId, Math.min(EARLY_CLOSE_SIGNAL_MS, operationTimeoutMs)); } catch (final WinRMFaultException e) { + retireShell(); if (!FAULT_OPERATION_TIMEOUT.equals(e.getFaultCode())) { - retireShell(); throw e; } } catch (final Exception e) { diff --git a/src/test/java/org/metricshub/winrm/StreamingApiTest.java b/src/test/java/org/metricshub/winrm/StreamingApiTest.java index f7d9fef..489ba8c 100644 --- a/src/test/java/org/metricshub/winrm/StreamingApiTest.java +++ b/src/test/java/org/metricshub/winrm/StreamingApiTest.java @@ -817,6 +817,20 @@ void aFailedEarlyCloseSignalRetiresTheShell() throws Exception { } } + @Test + void anEarlyCloseSignalHeldPastItsHoldRetiresTheShell() throws Exception { + enqueueCommandStartup(); + // The service does not finish processing the Signal within its short hold: not a failure of + // close(), but nothing proves the command is gone from SHELL-1. + server.enqueue(500, fault(FAULT_OPERATION_TIMEOUT, "The operation timed out.")); + enqueueNextCommandInANewShell(); + + try (WinRMClient client = builder().build()) { + client.executor().startCommand("run.exe", null, 10_000).close(); + assertNextCommandRunsInANewShell(client); + } + } + @Test void anEarlyCloseSignalLostInTransitRetiresTheShell() throws Exception { enqueueCommandStartup(); From 52223dc15080cd5d4313d017dd874703d4506676 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Fri, 9 Oct 2026 11:33:36 +0200 Subject: [PATCH 5/7] Drop the deadline workaround now that #198 fixed it in the transport Since #198, HttpTransport.connect() re-arms the streaming deadline on every explicit reconnect, so the Create that follows a timed-out retired-shell Delete no longer inherits an expired deadline. Remove the configureTimeouts() call that worked around it (its comment was no longer true) and keep the cancellation check. Also restore the blank line between the #196 and #198 CHANGELOG entries. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + src/main/java/org/metricshub/winrm/light/WsmanClient.java | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353ee70..4afed9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,7 @@ Consequences: of concurrent operations for this user has been exceeded*), until the client was recreated. Such a shell is now retired: the next command deletes it and runs in a fresh shell, created with the same working directory, environment and profile. + - **A streaming read resumed after a long pause no longer times out spuriously** (#198). When a streaming consumer (e.g. a `RemoteProcess` read slowly) paused longer than the inactivity timeout and the host dropped the idle connection meanwhile, the reconnection inherited the diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index 517a578..d7e8ec9 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -570,10 +570,8 @@ RemoteCommand startCommand( if (shellId == null) { if (retiredShellId != null) { deleteRetiredShell(operationTimeoutMs); - // The Delete was a round trip of its own: in streaming mode a timed-out one leaves - // the transport's per-leg deadline expired, which would give the Create's reconnect - // a 1 ms budget. And never create a shell after the caller's timeout was reported. - configureTimeouts(operationTimeoutMs, failOnQuietTimeout); + // The Delete was a round trip of its own: never create a shell after the caller's + // timeout was reported. checkNotCancelled(); } createShell(shellWorkingDirectory, shellEnvironment, operationTimeoutMs, failOnQuietTimeout); From 0665a0967b210f05edb91b1afcb6467c702e6131 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Fri, 9 Oct 2026 17:36:04 +0200 Subject: [PATCH 6/7] Replace the shell every N commands and when the quota refuses a command Address live tests on Windows Server 2008 R2 and 2022: every command run in a shell holds one of the user's WSMan operations until the shell is deleted, even when its terminate Signal succeeded; time does not release them. A long-lived client therefore reached MaxConcurrentOperationsPerUser (15 on 2008 R2, 1500 later) without any failed Signal, which is #196 too. - The client replaces its shell every maxCommandsPerShell commands: a new WinRMClient.Builder option, 10 by default, plumbed through a new LightWinRMService.createInstance overload (the previous one delegates with the default). WsmanClient counts the commands run in the current shell and retires it before the next command once the count is reached; the existing flow deletes it and creates a new one with the same working directory, environment and profile. - A Command the quota refuses in a shell that already ran commands retires and deletes that shell, then is retried once in a new one (it ran nothing). In a fresh shell the fault is reported. - The quota fault is recognized by its code 2150859174 or, on hosts that send no WSManFault element (a French Windows Server 2022), by its SOAP subcode, now exposed as WinRMFaultException.getFaultSubcode(). Docs: new "Shell reuse" section in commands.md (including that deleting a shell ends processes left running in it), client options row, fault detail table, host quota row, winrm4j migration guide, CHANGELOG. The ShellFileCopy quota-retry comments no longer claim the budget recovers over time. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 22 ++- .../org/metricshub/winrm/CommandRequest.java | 8 +- .../org/metricshub/winrm/ShellFileCopy.java | 11 +- .../org/metricshub/winrm/WinRMClient.java | 32 ++++ .../winrm/exceptions/WinRMFaultException.java | 36 ++++ .../winrm/light/LightWinRMService.java | 79 ++++++++ .../metricshub/winrm/light/WsmanClient.java | 89 +++++++-- src/site/markdown/commands.md | 26 ++- src/site/markdown/index.md | 1 + src/site/markdown/migrating-from-winrm4j.md | 15 +- src/site/markdown/preparing-the-host.md | 2 +- src/site/markdown/timeouts-and-errors.md | 1 + .../winrm/WinRMClientBuilderTest.java | 10 + .../org/metricshub/winrm/WinRMClientTest.java | 171 ++++++++++++++++++ .../winrm/light/BasicAuthCloseRaceTest.java | 1 + 15 files changed, 464 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4afed9b..c46dcff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -158,14 +158,20 @@ Consequences: ### Fixed -- **A long-lived client no longer exhausts the host's operation quota** (#196). When the - terminate `Signal` ending a command failed (a fault, a dropped connection) or was skipped (no - time left in a bounded poll), the client kept reusing the shell, and the command, never - terminated, kept holding one of the user's WSMan operations. A client polling indefinitely - piled them up until every command was refused with WSManFault 2150859174 (*the maximum number - of concurrent operations for this user has been exceeded*), until the client was recreated. - Such a shell is now retired: the next command deletes it and runs in a fresh shell, created - with the same working directory, environment and profile. +- **A long-lived client no longer exhausts the host's operation quota** (#196). Every command + run in a shell holds one of the user's WSMan operations until the shell is deleted, even once + terminated, and the client reused its shell forever: a client polling indefinitely ended up + with every command refused with WSManFault 2150859174 (*the maximum number of concurrent + operations for this user has been exceeded*), after 15 commands on Windows Server 2008 R2 and + 1500 on later versions, until it was recreated. The client now replaces its shell every 10 + commands (the new `WinRMClient.Builder.maxCommandsPerShell(int)` changes that number), retries + once in a new shell a command the quota refuses in a shell that already ran commands, and never + reuses a shell holding a command it could not terminate (its terminate `Signal` failed or was + skipped). The new shell gets the same working directory, environment and profile; deleting the + old one ends any process a previous command left running in it, as closing the client always + did. The quota fault is recognized by its code or, on hosts that send none (a French + Windows Server 2022), by its SOAP subcode, now exposed as + `WinRMFaultException.getFaultSubcode()`. - **A streaming read resumed after a long pause no longer times out spuriously** (#198). When a streaming consumer (e.g. a `RemoteProcess` read slowly) paused longer than the inactivity diff --git a/src/main/java/org/metricshub/winrm/CommandRequest.java b/src/main/java/org/metricshub/winrm/CommandRequest.java index 604d2e4..832917a 100644 --- a/src/main/java/org/metricshub/winrm/CommandRequest.java +++ b/src/main/java/org/metricshub/winrm/CommandRequest.java @@ -192,8 +192,9 @@ private String powerShellFromFile(final String script, final long timeoutMillis, /** * Set the working directory of the remote process. The remote command shell is created on - * the first command a client executes and is reused afterward, so this setting takes effect - * only when this is the client's first command. + * the first command a client executes and is reused afterward (every shell that replaces it + * gets the same settings), so this setting takes effect only when this is the client's first + * command. * * @param workingDirectory the working directory path on the remote host * @return this request @@ -209,7 +210,8 @@ public CommandRequest workingDirectory(final String workingDirectory) { * be called several times; insertion order is preserved, and setting the same name again * replaces its value. Like {@link #workingDirectory(String)}, the environment is shell-scoped: * the remote command shell is created on the first command a client executes and is reused - * afterward, so this setting takes effect only when this is the client's first command. + * afterward (every shell that replaces it gets the same settings), so this setting takes + * effect only when this is the client's first command. * *
{@code
 	 * CommandResult result = client.command("build.cmd")
diff --git a/src/main/java/org/metricshub/winrm/ShellFileCopy.java b/src/main/java/org/metricshub/winrm/ShellFileCopy.java
index 417d558..67da651 100644
--- a/src/main/java/org/metricshub/winrm/ShellFileCopy.java
+++ b/src/main/java/org/metricshub/winrm/ShellFileCopy.java
@@ -118,8 +118,10 @@ private ShellFileCopy() {}
 
 	/**
 	 * Base delay before retrying after an operation-quota rejection; each retry waits one step
-	 * longer. Measured on Windows 2008 R2 (quota 15 per user): the budget fully recovers within
-	 * 30 seconds, so the escalating delays (5+10+15+20 s) comfortably bridge it.
+	 * longer (5+10+15+20 s). Time does not release the operations a shell holds, only
+	 * deleting the shell does, and the client already replaces its own shell when the quota
+	 * refuses a command (see {@code WsmanClient}): these delays give the user's other
+	 * connections time to release theirs.
 	 */
 	static final long QUOTA_RETRY_DELAY_MILLIS = 5_000L;
 
@@ -796,8 +798,9 @@ private static WindowsRemoteCommandResult run(
 
 				// The quota rejection happened while the operation was being CREATED — before the
 				// command could run — so retrying cannot duplicate a side effect. Old Windows
-				// versions cap concurrent operations very low (15 per user on 2008 R2) and reap
-				// completed ones lazily: give the server increasingly more time to recover.
+				// versions cap concurrent operations very low (15 per user on 2008 R2), and the
+				// client already replaced its own shell: give the user's other connections
+				// increasingly more time to release theirs.
 				try {
 					Utils.sleep(
 						Math.min(
diff --git a/src/main/java/org/metricshub/winrm/WinRMClient.java b/src/main/java/org/metricshub/winrm/WinRMClient.java
index e49daed..91eea7c 100644
--- a/src/main/java/org/metricshub/winrm/WinRMClient.java
+++ b/src/main/java/org/metricshub/winrm/WinRMClient.java
@@ -331,6 +331,7 @@ public static final class Builder {
 		private int consoleCodePage;
 		private boolean loadUserProfile;
 		private String arraySeparator = LightWinRMService.DEFAULT_ARRAY_SEPARATOR;
+		private int maxCommandsPerShell = LightWinRMService.DEFAULT_MAX_COMMANDS_PER_SHELL;
 		private SSLContext sslContext;
 		private Duration timeout = DEFAULT_TIMEOUT;
 		private int retries;
@@ -624,6 +625,36 @@ public Builder arraySeparator(final String arraySeparator) {
 			return this;
 		}
 
+		/**
+		 * Set how many commands the remote command shell runs before the client replaces it.
+		 * Default: 10.
+		 * 

+ * The client reuses its command shell for commands, file transfers and remote file + * operations, but every command run in a shell holds one of the user's WSMan operations + * until the shell is deleted, even after it completed. The host caps them per user, + * across all of that user's connections ({@code MaxConcurrentOperationsPerUser}: 15 on + * Windows Server 2008 R2, 1500 later), so a shell reused forever ends up having every + * command refused. Replacing the shell (one Delete and one Create, about 100 ms) releases + * them. The new shell gets the same working directory, environment and profile, but + * deleting the old one ends any process a previous command left running in it, as + * {@link WinRMClient#close()} does. Besides, a command the quota refuses in a shell that + * already ran commands is retried once in a new shell; in a fresh shell, which holds + * nothing to release, the fault is reported. + *

+ * A lower value leaves more of the quota to the user's other connections at the cost of + * more frequent replacements; 1 runs every command in a shell of its own, like {@code winrs}. + * + * @param maxCommandsPerShell how many commands a shell runs before it is replaced (at least 1) + * @return this builder + */ + public Builder maxCommandsPerShell(final int maxCommandsPerShell) { + if (maxCommandsPerShell < 1) { + throw new IllegalArgumentException("maxCommandsPerShell must be at least 1."); + } + this.maxCommandsPerShell = maxCommandsPerShell; + return this; + } + /** * Build the client. This does not connect yet: the connection is established and * authenticated by the first operation. @@ -672,6 +703,7 @@ public WinRMClient build() { consoleCodePage, loadUserProfile, arraySeparator, + maxCommandsPerShell, retries, toMillis(retryDelay) ); diff --git a/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java b/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java index e72a00b..99e64c2 100644 --- a/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java +++ b/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java @@ -35,15 +35,37 @@ public class WinRMFaultException extends WinRMClientException { private final int httpStatus; private final String faultCode; + private final String faultSubcode; private final String faultReason; private final String faultDetail; + /** + * Create the exception, without a SOAP fault subcode. + * + * @param message the complete detail message (same format as the legacy API) + * @param httpStatus the HTTP status of the faulting response + * @param faultCode the WSManFault code, or {@code null} when the response carried none + * @param faultReason the SOAP fault reason text, or {@code null} + * @param faultDetail the detailed WSManFault message (provider-level detail), or {@code null} + */ + public WinRMFaultException( + final String message, + final int httpStatus, + final String faultCode, + final String faultReason, + final String faultDetail + ) { + this(message, httpStatus, faultCode, null, faultReason, faultDetail); + } + /** * Create the exception. * * @param message the complete detail message (same format as the legacy API) * @param httpStatus the HTTP status of the faulting response * @param faultCode the WSManFault code, or {@code null} when the response carried none + * @param faultSubcode the SOAP fault subcode without its namespace prefix (e.g. + * {@code QuotaLimit}), or {@code null} when the response carried none * @param faultReason the SOAP fault reason text, or {@code null} * @param faultDetail the detailed WSManFault message (provider-level detail), or {@code null} */ @@ -51,12 +73,14 @@ public WinRMFaultException( final String message, final int httpStatus, final String faultCode, + final String faultSubcode, final String faultReason, final String faultDetail ) { super(message); this.httpStatus = httpStatus; this.faultCode = faultCode; + this.faultSubcode = faultSubcode; this.faultReason = faultReason; this.faultDetail = faultDetail; } @@ -79,6 +103,18 @@ public String getFaultCode() { return faultCode; } + /** + * Get the SOAP fault subcode, without its namespace prefix: the WS-Management fault name, such + * as {@code QuotaLimit} or {@code InvalidSelectors}. Unlike the reason text it is never + * translated, and some hosts send it without any WSManFault code (e.g. the quota fault of a + * French Windows Server 2022). + * + * @return the fault subcode, or {@code null} when the response carried none + */ + public String getFaultSubcode() { + return faultSubcode; + } + /** * Get the SOAP fault reason text. * diff --git a/src/main/java/org/metricshub/winrm/light/LightWinRMService.java b/src/main/java/org/metricshub/winrm/light/LightWinRMService.java index 828f80a..862e972 100644 --- a/src/main/java/org/metricshub/winrm/light/LightWinRMService.java +++ b/src/main/java/org/metricshub/winrm/light/LightWinRMService.java @@ -62,6 +62,13 @@ public final class LightWinRMService implements WindowsRemoteExecutor { /** The default string joining the elements of a WMI array property in a WQL row. */ public static final String DEFAULT_ARRAY_SEPARATOR = "|"; + /** + * The default number of commands a remote command shell runs before the client replaces it. + * Each command holds one of the user's WSMan operations until its shell is deleted, and + * Windows Server 2008 R2 allows only 15 per user ({@code MaxConcurrentOperationsPerUser}). + */ + public static final int DEFAULT_MAX_COMMANDS_PER_SHELL = 10; + private final WinRMEndpoint winRMEndpoint; private final WsmanClient client; private final AtomicBoolean closed = new AtomicBoolean(false); @@ -275,6 +282,71 @@ public static LightWinRMService createInstance( * Create a light WinRM executor that may delegate the caller's Kerberos credentials to the host, * so remote commands can authenticate onward as the caller (the second hop), may load the * user profile in the command shell, and joins WMI array properties with a custom separator. + * The command shell is replaced every {@link #DEFAULT_MAX_COMMANDS_PER_SHELL} commands. + * + * @param winRMEndpoint endpoint with credentials (mandatory) + * @param timeout timeout in milliseconds (must be > 0) + * @param ticketCache Kerberos ticket cache path (used by the Kerberos scheme; {@code null} logs + * in with the password) + * @param authentications requested authentication schemes, tried in order (NTLM, Kerberos, and/or Basic); + * {@code null}/empty means NTLM only + * @param allowDelegation whether Kerberos forwards the caller's ticket-granting ticket to the + * host (which must then be forwardable); requires Kerberos among {@code authentications} + * @param sslContext the {@link SSLContext} providing the HTTPS socket factory (hostname + * verification stays on); {@code null} uses the default configuration + * @param trustAllCertificates when {@code true} (and no {@code sslContext} is given), trust every + * server certificate and skip hostname verification — insecure, testing only + * @param consoleCodePage the console code page of the command shell; 0 keeps the default 65001, + * which makes command output UTF-8 whatever the remote locale + * @param loadUserProfile whether the command shell loads the user profile (registry hive, + * per-user environment variables); {@code false} is the historical behavior + * @param arraySeparator the string joining the elements of a WMI array property in a WQL row; + * see {@link #DEFAULT_ARRAY_SEPARATOR} + * @param connectRetries how many times one round trip may re-attempt to connect and authenticate + * (must be >= 0); 0 keeps the historical fail-fast behavior + * @param retryDelay the pause in milliseconds before each retry (must be >= 0) + * @return a new {@code LightWinRMService} + * @throws WinRMException on invalid arguments or an unsupported authentication request + */ + // CPD-OFF — a compatibility overload: its parameter list is the next overload's minus the + // shell bound, and reordering the parameters to fool the detector would break callers. + public static LightWinRMService createInstance( + final WinRMEndpoint winRMEndpoint, + final long timeout, + final java.nio.file.Path ticketCache, + final List authentications, + final boolean allowDelegation, + final SSLContext sslContext, + final boolean trustAllCertificates, + final int consoleCodePage, + final boolean loadUserProfile, + final String arraySeparator, + final int connectRetries, + final long retryDelay + ) throws WinRMException { + return createInstance( + winRMEndpoint, + timeout, + ticketCache, + authentications, + allowDelegation, + sslContext, + trustAllCertificates, + consoleCodePage, + loadUserProfile, + arraySeparator, + DEFAULT_MAX_COMMANDS_PER_SHELL, + connectRetries, + retryDelay + ); + // CPD-ON + } + + /** + * Create a light WinRM executor that may delegate the caller's Kerberos credentials to the host, + * so remote commands can authenticate onward as the caller (the second hop), may load the + * user profile in the command shell, joins WMI array properties with a custom separator, and + * replaces the command shell after a given number of commands. * * @param winRMEndpoint endpoint with credentials (mandatory) * @param timeout timeout in milliseconds (must be > 0) @@ -294,6 +366,8 @@ public static LightWinRMService createInstance( * per-user environment variables); {@code false} is the historical behavior * @param arraySeparator the string joining the elements of a WMI array property in a WQL row; * see {@link #DEFAULT_ARRAY_SEPARATOR} + * @param maxCommandsPerShell how many commands a command shell runs before it is replaced (must + * be > 0); see {@link #DEFAULT_MAX_COMMANDS_PER_SHELL} * @param connectRetries how many times one round trip may re-attempt to connect and authenticate * (must be >= 0); 0 keeps the historical fail-fast behavior * @param retryDelay the pause in milliseconds before each retry (must be >= 0) @@ -311,12 +385,16 @@ public static LightWinRMService createInstance( final int consoleCodePage, final boolean loadUserProfile, final String arraySeparator, + final int maxCommandsPerShell, final int connectRetries, final long retryDelay ) throws WinRMException { Utils.checkNonNull(winRMEndpoint, "winRMEndpoint"); Utils.checkNonNull(arraySeparator, "arraySeparator"); Utils.checkArgumentNotZeroOrNegative(timeout, "timeout"); + if (maxCommandsPerShell < 1) { + throw new IllegalArgumentException("maxCommandsPerShell must be at least 1."); + } if (connectRetries < 0) { throw new IllegalArgumentException("connectRetries must not be negative."); } @@ -366,6 +444,7 @@ public static LightWinRMService createInstance( consoleCodePage, loadUserProfile, arraySeparator, + maxCommandsPerShell, connectRetries, retryDelay ); diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index d7e8ec9..4ebf13f 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -20,6 +20,7 @@ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ */ +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -58,6 +59,14 @@ final class WsmanClient implements AutoCloseable { private static final String FAULT_OPERATION_TIMEOUT = "2150858793"; private static final String FAULT_SHELL_NOT_FOUND = "2150858843"; + // The user's MaxConcurrentOperationsPerUser quota is full. Some hosts (a French Windows Server + // 2022) send this fault without any WSManFault code, so it is also recognized by its SOAP + // subcode, which is never translated. + private static final String FAULT_OPERATION_QUOTA = "2150859174"; + private static final String SUBCODE_QUOTA_LIMIT = "QuotaLimit"; + + private static final String SOAP_ENVELOPE_NS = "http://www.w3.org/2003/05/soap-envelope"; + // A Send answered with this Windows error (ERROR_NO_DATA, "The pipe is being closed") found the // command no longer reading its standard input: it exited, or closed its stdin, first. private static final String FAULT_PIPE_CLOSING = "232"; @@ -92,6 +101,7 @@ final class WsmanClient implements AutoCloseable { private final int consoleCodePage; private final boolean loadUserProfile; private final String arraySeparator; + private final int maxCommandsPerShell; private final String url; private final String rawUsername; private final AuthScheme auth; @@ -107,14 +117,23 @@ final class WsmanClient implements AutoCloseable { private String pendingAuthorization; private String shellId; - // A shell no longer reused because one of its commands may never have been terminated: its - // terminate Signal faulted, failed in transit, or found no budget (issue #196). Such a command - // keeps holding one of the user's WSMan operations until its shell is deleted, so reusing the - // shell would pile them up until MaxConcurrentOperationsPerUser refuses every command. Deleted - // before the next shell is created, or by close(); never set together with shellId. Guarded by - // connectionPermit, like shellId. + // A shell no longer reused: it ran maxCommandsPerShell commands, the quota refused a command in + // it, or one of its commands may never have been terminated (its terminate Signal faulted, failed + // in transit, or found no budget). Its commands hold the user's WSMan operations until it is + // deleted, so reusing it would pile them up until MaxConcurrentOperationsPerUser refuses every + // command (issue #196). Deleted before the next shell is created, or by close(); never set + // together with shellId. Guarded by connectionPermit, like shellId. private String retiredShellId; + // How many commands the current shell has run. Each one holds one of the user's WSMan operations + // until the shell is deleted, even once cleanly terminated (measured on Windows Server 2008 R2 + // and 2022, issue #196), so the shell is retired after maxCommandsPerShell of them. Guarded by + // connectionPermit, like shellId. + @SuppressFBWarnings(value = "AT_STALE_THREAD_WRITE_OF_PRIMITIVE", justification = "Only accessed while holding connectionPermit, " + + + "whose release/acquire orders every write before the next holder's reads") + private int shellCommands; + // The shell's working directory and environment variables are pinned by the FIRST command on // this connection and reused whenever the shell must be (re)created — e.g. after the server // reaped it — so a recreation stays invisible to the caller instead of silently moving later @@ -199,6 +218,7 @@ private static void checkNotCancelled() throws InterruptedException { final int consoleCodePage, final boolean loadUserProfile, final String arraySeparator, + final int maxCommandsPerShell, final int connectRetries, final long retryDelayMs ) { @@ -206,6 +226,7 @@ private static void checkNotCancelled() throws InterruptedException { this.consoleCodePage = consoleCodePage; this.loadUserProfile = loadUserProfile; this.arraySeparator = arraySeparator; + this.maxCommandsPerShell = maxCommandsPerShell; this.connectRetries = connectRetries; this.retryDelayMs = retryDelayMs; // A non-null socket factory selects HTTPS: TLS wraps the transport and the SOAP travels plaintext. @@ -567,6 +588,11 @@ RemoteCommand startCommand( : new LinkedHashMap<>(environment); shellSettingsPinned = true; } + if (shellId != null && shellCommands >= maxCommandsPerShell) { + // The shell's commands hold as many of the user's WSMan operations: replace it before + // they exhaust the quota shared by every connection of this user. + retireShell(); + } if (shellId == null) { if (retiredShellId != null) { deleteRetiredShell(operationTimeoutMs); @@ -583,18 +609,26 @@ RemoteCommand startCommand( try { commandId = sendCommand(commandLine, operationTimeoutMs, failOnQuietTimeout, consoleModeStdin); } catch (final WinRMFaultException e) { - if (!FAULT_SHELL_NOT_FOUND.equals(e.getFaultCode())) { + // Either way the Command was rejected before it could run, so it is safe to recreate + // the shell — with its ORIGINAL working directory and environment — and retry once. + if (FAULT_SHELL_NOT_FOUND.equals(e.getFaultCode())) { + // The server reaped the cached shell between commands (e.g. its IdleTimeout expired on + // a long-lived client). + shellId = null; + } else if (shellCommands > 0 && isQuotaFault(e)) { + // The user's operation quota is full, and this shell holds one operation per command + // it ran: deleting it releases them. + retireShell(); + deleteRetiredShell(operationTimeoutMs); + checkNotCancelled(); + } else { throw e; } - // The server reaped the cached shell between commands (e.g. its IdleTimeout expired on a - // long-lived client). The Command was rejected before it could run, so it is safe to - // recreate the shell — with its ORIGINAL working directory and environment — and retry - // once. - shellId = null; createShell(shellWorkingDirectory, shellEnvironment, operationTimeoutMs, failOnQuietTimeout); checkNotCancelled(); commandId = sendCommand(commandLine, operationTimeoutMs, failOnQuietTimeout, consoleModeStdin); } + shellCommands++; opened = true; return new RemoteCommand(commandId, operationTimeoutMs, failOnQuietTimeout); } finally { @@ -1036,6 +1070,7 @@ private void createShell( final Element selector = (Element) selectors.item(i); if ("ShellId".equals(selector.getAttribute("Name"))) { shellId = selector.getTextContent(); + shellCommands = 0; return; } } @@ -1084,10 +1119,9 @@ private void signal(final String commandId, final String code, final long timeou } /** - * Stop reusing the current shell: one of its commands may never have been terminated, and only - * deleting the shell releases the WSMan operation that command holds (issue #196). The Delete - * is deferred to the next shell creation or to {@link #close()}: the failed or skipped Signal - * that got us here leaves no budget, and possibly no connection, to send it now. + * Stop reusing the current shell: only deleting it releases the WSMan operations its commands + * hold (issue #196). The Delete is deferred to the next shell creation or to {@link #close()}: + * a failed or skipped Signal leaves no budget, and possibly no connection, to send it now. */ private void retireShell() { if (shellId != null) { @@ -1185,6 +1219,7 @@ private static WinRMFaultException faultException(final String operation, final operation + " failed: " + faultSummary(resp), resp.status, trimToNull(wsmanFaultCode(resp.document)), + trimToNull(faultSubcode(resp.document)), trimToNull(text(resp.document, "Text")), trimToNull(wsmanFaultMessage(resp.document)) ); @@ -1456,6 +1491,28 @@ private static String wsmanFaultCode(final Document doc) { return faults.getLength() > 0 ? ((Element) faults.item(0)).getAttribute("Code") : null; } + /** + * The SOAP fault subcode without its namespace prefix (e.g. {@code QuotaLimit}), or null: the + * fault's name, never translated, which some hosts send without any WSManFault code. + */ + private static String faultSubcode(final Document doc) { + final NodeList subcodes = doc.getElementsByTagNameNS(SOAP_ENVELOPE_NS, "Subcode"); + if (subcodes.getLength() == 0) { + return null; + } + final NodeList values = ((Element) subcodes.item(0)).getElementsByTagNameNS(SOAP_ENVELOPE_NS, "Value"); + if (values.getLength() == 0) { + return null; + } + final String value = values.item(0).getTextContent().trim(); + return value.substring(value.indexOf(':') + 1); + } + + /** Whether the fault says the user's WSMan operation quota (MaxConcurrentOperationsPerUser) is full. */ + private static boolean isQuotaFault(final WinRMFaultException fault) { + return FAULT_OPERATION_QUOTA.equals(fault.getFaultCode()) || SUBCODE_QUOTA_LIMIT.equals(fault.getFaultSubcode()); + } + /** * The detailed WSManFault Message text, or null. This is where WinRM puts the provider-level * detail — notably the WMI error mnemonics (WBEM_E_INVALID_CLASS, WBEM_E_INVALID_NAMESPACE, diff --git a/src/site/markdown/commands.md b/src/site/markdown/commands.md index 2d4dd62..1a79a34 100644 --- a/src/site/markdown/commands.md +++ b/src/site/markdown/commands.md @@ -75,8 +75,8 @@ try (WinRMClient client = WinRMClient.builder("server.example.com") The profile is loaded when the remote shell is created, so this is a client setting. It applies to every shell the client creates, for commands, file transfers and remote file operations alike, -including a shell recreated after the server reaped the previous one, or after the client deleted -one holding a command it could not terminate. Microsoft's `winrs` +including a shell recreated after the server reaped the previous one or the client replaced it +(see [Shell reuse](#shell-reuse)). Microsoft's `winrs` documentation warns that loading the profile fails for a user who is not a local administrator on the host: the command then fails with a [`WinRMFaultException`](apidocs/org/metricshub/winrm/exceptions/WinRMFaultException.html) carrying @@ -86,6 +86,28 @@ A command that reaches a further host (a UNC path, another server) fails with *a unless the client delegates your Kerberos credentials: see [Credential delegation](authentication.html#credential-delegation). +### Shell reuse + +The client runs its commands, file transfers and remote file operations in one remote command +shell, created by the first of them. But every command run in a shell holds one of the user's +WSMan operations until the shell is deleted, even after it completed, and the host caps them per +user, across all of that user's connections: `MaxConcurrentOperationsPerUser` is 15 on Windows +Server 2008 R2 and 1500 later (see [Host quotas](preparing-the-host.html#host-quotas-worth-knowing-about)). +So the client replaces its shell: + +* every 10 commands. The builder's `maxCommandsPerShell(int)` changes that number: lower leaves + more of the quota to the user's other connections, and 1 runs every command in a shell of its + own, like `winrs`; +* when the quota refuses a command in a shell that already ran commands: the command ran nothing, + so it is retried once in a new shell. In a fresh shell, which holds nothing to release, the fault + is reported; +* when a command could not be terminated cleanly (its terminate `Signal` failed or was skipped). + +A replacement costs a Delete and a Create (about 100 ms). The new shell gets the same working +directory, environment variables and profile, but deleting the old one ends any process a previous +command left running in it, as closing the client does. A process that must outlive its command +belongs outside the shell, e.g. in a scheduled task. + ## Running PowerShell `powerShell(...)` prepares a PowerShell script execution the same way `command(...)` prepares a diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md index 7a2b761..104b0d0 100644 --- a/src/site/markdown/index.md +++ b/src/site/markdown/index.md @@ -151,6 +151,7 @@ Besides the host name passed to `builder(...)`, only `credentials(...)` is manda | `retries(int, Duration)` | no retry | [Retrying](timeouts-and-errors.html#retrying-transient-connection-failures) | | `namespace(String)` | `ROOT\CIMV2` | [Choosing a namespace](wql.html#choosing-a-namespace) | | `loadUserProfile()` | not loaded | [Loading the user profile](commands.html#loading-the-user-profile) | +| `maxCommandsPerShell(int)` | 10 | [Shell reuse](commands.html#shell-reuse) | | `consoleCodePage(int)` | 65001 (UTF-8) | [Input encoding](commands.html#input-encoding) | | `arraySeparator(String)` | `\|` | [Reading the result](wql.html#reading-the-result) | diff --git a/src/site/markdown/migrating-from-winrm4j.md b/src/site/markdown/migrating-from-winrm4j.md index 0f87d40..639d758 100644 --- a/src/site/markdown/migrating-from-winrm4j.md +++ b/src/site/markdown/migrating-from-winrm4j.md @@ -158,12 +158,15 @@ the switch: to retry `Receive` after an operation timeout. Here nothing is retried unless you opt in with `retries(int, Duration)`, and the policy is deliberately narrow: only attempts that provably never reached the server (TCP connect, DNS, TLS handshake, the authentication handshake) are - retried, preserving **at-most-once execution** for non-idempotent commands. A request that was - actually sent is never replayed. -* **One shell per client.** winrm4j creates and tears down a remote shell for every - `executeCommand(...)`. This client creates the shell on the first command and reuses it, which - is faster — and is why `workingDirectory(...)` and `environment(...)` are per-command options - that take effect on the client's **first** command ([command options](commands.html#command-options)). + retried, preserving **at-most-once execution** for non-idempotent commands. A request that may + have run is never replayed: the only replays are of a command the host refused before running it + (its shell was reaped, or the quota was full; see [Shell reuse](commands.html#shell-reuse)). +* **One shell for many commands.** winrm4j creates and tears down a remote shell for every + `executeCommand(...)`. This client creates the shell on the first command and reuses it, for 10 + commands by default ([Shell reuse](commands.html#shell-reuse)), which is faster — and is why + `workingDirectory(...)` and `environment(...)` are per-command options that take effect on the + client's **first** command, then on every shell that replaces it + ([command options](commands.html#command-options)). ## What you gain diff --git a/src/site/markdown/preparing-the-host.md b/src/site/markdown/preparing-the-host.md index a3a5fed..2905aaf 100644 --- a/src/site/markdown/preparing-the-host.md +++ b/src/site/markdown/preparing-the-host.md @@ -369,7 +369,7 @@ the host, the tighter they are: | --- | --- | --- | | `MaxMemoryPerShellMB` | Memory per shell, including child processes | Historically **150 MB**; 1024 MB on modern hosts. A command whose output is large can hit it. | | `MaxShellsPerUser` | Concurrent shells per user | 5 on older hosts, 30 on modern ones. Close clients you no longer need. | -| `MaxConcurrentOperationsPerUser` | Concurrent operations per user | 15 on Windows Server 2008 R2, 1500 later. File transfers are batched specifically to stay under low limits. A command holds one until it is terminated; when the client cannot terminate it, it deletes that shell rather than reuse it, so a long-lived client does not pile them up. | +| `MaxConcurrentOperationsPerUser` | Concurrent operations per user | 15 on Windows Server 2008 R2, 1500 later. File transfers are batched specifically to stay under low limits. Every command holds one until its shell is deleted, even after it completed: the client replaces its shell every 10 commands and when the quota refuses a command in a shell that already ran commands, so a long-lived client does not pile them up (see [Shell reuse](commands.html#shell-reuse)). | | `MaxEnvelopeSizekb` | SOAP envelope size | 150 KB on older hosts, **500 KB** on modern ones. This client always uses 150 KB envelopes, so the default never needs raising. | | `IdleTimeout` | How long an idle shell survives | 180000 ms (3 min) on older hosts, **7200000 ms** (2 h) on modern ones; 60000 ms minimum. The client transparently recreates a shell the host reaped. | diff --git a/src/site/markdown/timeouts-and-errors.md b/src/site/markdown/timeouts-and-errors.md index 539b8bb..789403a 100644 --- a/src/site/markdown/timeouts-and-errors.md +++ b/src/site/markdown/timeouts-and-errors.md @@ -115,6 +115,7 @@ the fault **programmatically**, so no message parsing is needed: | Method | Returns | | --- | --- | | `getFaultCode()` | The WSManFault code as a decimal `String`, e.g. `"2150858778"`, or `null` when the response carried none. | +| `getFaultSubcode()` | The SOAP fault subcode without its prefix, e.g. `"QuotaLimit"`, or `null`. Never translated, and present on hosts that send no WSManFault code (the quota fault of a French Windows Server 2022). | | `getFaultReason()` | The SOAP fault reason text, or `null`. | | `getFaultDetail()` | The provider-level detail — where WMI puts mnemonics such as `WBEM_E_INVALID_CLASS` or `WBEM_E_INVALID_NAMESPACE` — or `null`. | | `getHttpStatus()` | The HTTP status of the faulting response (typically 500). | diff --git a/src/test/java/org/metricshub/winrm/WinRMClientBuilderTest.java b/src/test/java/org/metricshub/winrm/WinRMClientBuilderTest.java index 31e8e93..7925e95 100644 --- a/src/test/java/org/metricshub/winrm/WinRMClientBuilderTest.java +++ b/src/test/java/org/metricshub/winrm/WinRMClientBuilderTest.java @@ -99,6 +99,16 @@ void retriesMustBeNonNegativeWithANonNegativeDelay() { } } + @Test + void maxCommandsPerShellMustBeAtLeastOne() { + assertThrows(IllegalArgumentException.class, () -> validBuilder().maxCommandsPerShell(0)); + assertThrows(IllegalArgumentException.class, () -> validBuilder().maxCommandsPerShell(-1)); + // 1 is valid: every command in a shell of its own. + try (WinRMClient client = validBuilder().maxCommandsPerShell(1).build()) { + assertEquals("host", client.hostname()); + } + } + @Test void authenticationMustNotBeEmpty() { assertThrows(IllegalArgumentException.class, () -> validBuilder().authentication()); diff --git a/src/test/java/org/metricshub/winrm/WinRMClientTest.java b/src/test/java/org/metricshub/winrm/WinRMClientTest.java index 653db7d..6fed5a1 100644 --- a/src/test/java/org/metricshub/winrm/WinRMClientTest.java +++ b/src/test/java/org/metricshub/winrm/WinRMClientTest.java @@ -23,10 +23,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.metricshub.winrm.light.FakeWsmanResponses.commandResponse; import static org.metricshub.winrm.light.FakeWsmanResponses.done; +import static org.metricshub.winrm.light.FakeWsmanResponses.enqueueShellDeletion; import static org.metricshub.winrm.light.FakeWsmanResponses.envelope; import static org.metricshub.winrm.light.FakeWsmanResponses.enumerationDone; import static org.metricshub.winrm.light.FakeWsmanResponses.fault; @@ -67,6 +69,20 @@ class WinRMClientTest { private static final String WSEN = "http://schemas.xmlsoap.org/ws/2004/09/enumeration"; private static final String WSMAN = "http://schemas.dmtf.org/wbem/wsman/1/wsman.xsd"; + /** + * What a French Windows Server 2022 answers when the user's operation quota is full: a SOAP + * fault with the {@code QuotaLimit} subcode and a translated reason, but no WSManFault element. + */ + private static final String QUOTA_FAULT_WITHOUT_CODE = "s:Receiver" + + "w:QuotaLimit" + + "Le nombre maximal d'opérations simultanées pour cet utilisateur a été dépassé." + + + ""; + private FakeWsmanServer server; @BeforeEach @@ -901,4 +917,159 @@ void closedClientRejectsOperationsAndCloseIsIdempotent() { client.close(); assertThrows(IllegalStateException.class, () -> client.wql("SELECT Name FROM Win32_Service").execute()); } + + // --- Shell replacement (issue #196) ------------------------------------------- + // + // Every command run in a shell holds one of the user's WSMan operations until the shell is + // deleted, even once cleanly terminated (measured on Windows Server 2008 R2 and 2022): a shell + // reused forever ends up having every command refused by MaxConcurrentOperationsPerUser. + + /** Script a whole command printing the given output, in the current shell. */ + private void enqueueCommand(final String commandId, final String stdout) { + server + .enqueue(200, envelope(commandResponse(commandId))) + .enqueue( + 200, + envelope( + receiveResponse(stream("stdout", commandId, stdout.getBytes(StandardCharsets.UTF_8)), done(commandId, 0)) + ) + ) + .enqueue(200, envelope(signalResponse())); + } + + /** Script the Delete of the replaced shell, then the creation of its replacement. */ + private void enqueueShellReplacement(final String newShell) { + enqueueShellDeletion(server); + server.enqueue(200, envelope(resourceCreated(newShell))); + } + + /** Assert that the requests from the given index delete SHELL-1, create a shell and use SHELL-2. */ + private void assertShellReplacedAt(final int index) { + assertShellReplacedAt(index, "SHELL-1", "SHELL-2"); + } + + /** Assert that the requests from the given index delete the old shell, create one and use the new one. */ + private void assertShellReplacedAt(final int index, final String oldShell, final String newShell) { + final List requests = server.decryptedRequests(); + final String delete = requests.get(index); + assertTrue(delete.contains("transfer/Delete"), delete); + assertTrue(delete.contains("Selector Name=\"ShellId\">" + oldShell + "<"), delete); + assertTrue(requests.get(index + 1).contains("transfer/Create"), requests.get(index + 1)); + final String command = requests.get(index + 2); + assertTrue(command.contains("Selector Name=\"ShellId\">" + newShell + "<"), command); + } + + @Test + void theShellIsReplacedAfterMaxCommandsPerShell() throws Exception { + server.enqueue(200, envelope(resourceCreated("SHELL-1"))); + enqueueCommand("CMD-1", "first"); + enqueueCommand("CMD-2", "second"); + enqueueShellReplacement("SHELL-2"); + enqueueCommand("CMD-3", "third"); + enqueueCommand("CMD-4", "fourth"); + enqueueShellReplacement("SHELL-3"); + enqueueCommand("CMD-5", "fifth"); + + try (WinRMClient client = builder(PASSWORD).maxCommandsPerShell(2).build()) { + for (final String output : List.of("first", "second", "third", "fourth", "fifth")) { + assertEquals(output, client.command(output + ".exe").execute().stdout()); + } + } + + // Create, then two whole commands (Command, Receive, Signal) in SHELL-1... + assertShellReplacedAt(7); + // ...and the count starts over in SHELL-2: two more commands before the next replacement. + assertShellReplacedAt(15, "SHELL-2", "SHELL-3"); + } + + @Test + void theShellIsReplacedAfterTenCommandsByDefault() throws Exception { + server.enqueue(200, envelope(resourceCreated("SHELL-1"))); + for (int i = 1; i <= 10; i++) { + enqueueCommand("CMD-" + i, "out"); + } + enqueueShellReplacement("SHELL-2"); + enqueueCommand("CMD-11", "out"); + + try (WinRMClient client = builder(PASSWORD).build()) { + for (int i = 0; i < 11; i++) { + assertEquals("out", client.command("poll.exe").execute().stdout()); + } + } + + // Create, then ten whole commands (Command, Receive, Signal) in SHELL-1. + assertShellReplacedAt(31); + } + + @Test + void aCommandRefusedByTheQuotaIsRetriedInANewShell() throws Exception { + assertQuotaFaultReplacesTheShell( + fault( + "2150859174", + "The WS-Management service cannot process the request. The maximum number of concurrent operations for this user has been exceeded." + ) + ); + } + + @Test + void aQuotaFaultWithoutWsmanFaultCodeIsRecognizedByItsSubcode() throws Exception { + assertQuotaFaultReplacesTheShell(QUOTA_FAULT_WITHOUT_CODE); + } + + private void assertQuotaFaultReplacesTheShell(final String quotaFault) throws Exception { + server.enqueue(200, envelope(resourceCreated("SHELL-1"))); + enqueueCommand("CMD-1", "first"); + // The user's operation quota is full: the second Command is refused before it could run. + server.enqueue(500, quotaFault); + enqueueShellReplacement("SHELL-2"); + enqueueCommand("CMD-2", "second"); + + try (WinRMClient client = builder(PASSWORD).build()) { + assertEquals("first", client.command("first.exe").execute().stdout()); + // SHELL-1 holds an operation per command it ran: deleting it releases them, and the + // refused command runs in a new shell. + assertEquals("second", client.command("second.exe").execute().stdout()); + } + + // Create, the first command, the refused Command in SHELL-1, then the replacement. + final String refused = server.decryptedRequests().get(4); + assertTrue(refused.contains("Selector Name=\"ShellId\">SHELL-1<"), refused); + assertShellReplacedAt(5); + } + + @Test + void aQuotaFaultOnAFreshShellIsReportedWithItsSubcode() throws Exception { + // The new shell ran nothing: deleting it would release nothing, so the fault is reported. + server.enqueue(200, envelope(resourceCreated("SHELL-1"))).enqueue(500, QUOTA_FAULT_WITHOUT_CODE); + + try (WinRMClient client = builder(PASSWORD).build()) { + final WinRMFaultException e = assertThrows(WinRMFaultException.class, () -> client.command("x.exe").execute()); + // No WSManFault code on such hosts: the untranslated subcode still names the fault. + assertNull(e.getFaultCode()); + assertEquals("QuotaLimit", e.getFaultSubcode()); + assertEquals(2, server.decryptedRequests().size(), "no Delete, no retry"); + } + } + + @Test + void aCommandRefusedByTheQuotaIsRetriedOnlyOnce() throws Exception { + // The quota is held by other connections of the user: the new shell is refused too, and + // the fault is reported instead of looping. + server.enqueue(200, envelope(resourceCreated("SHELL-1"))); + enqueueCommand("CMD-1", "first"); + server.enqueue(500, QUOTA_FAULT_WITHOUT_CODE); + enqueueShellReplacement("SHELL-2"); + server.enqueue(500, QUOTA_FAULT_WITHOUT_CODE); + + try (WinRMClient client = builder(PASSWORD).build()) { + assertEquals("first", client.command("first.exe").execute().stdout()); + final WinRMFaultException e = assertThrows( + WinRMFaultException.class, + () -> client.command("second.exe").execute() + ); + assertEquals("QuotaLimit", e.getFaultSubcode()); + final long commands = server.decryptedRequests().stream().filter(r -> r.contains(":CommandLine>")).count(); + assertEquals(3, commands, "the first command, the refused one, and a single retry"); + } + } } diff --git a/src/test/java/org/metricshub/winrm/light/BasicAuthCloseRaceTest.java b/src/test/java/org/metricshub/winrm/light/BasicAuthCloseRaceTest.java index 2cb8216..4964128 100644 --- a/src/test/java/org/metricshub/winrm/light/BasicAuthCloseRaceTest.java +++ b/src/test/java/org/metricshub/winrm/light/BasicAuthCloseRaceTest.java @@ -49,6 +49,7 @@ void closeWhileOperationInFlightStillErasesTheBasicCredential() throws Exception 65001, false, "|", + LightWinRMService.DEFAULT_MAX_COMMANDS_PER_SHELL, 0, 0L ); From 371e3c8be938ac99841c1f0e31200b5323399e13 Mon Sep 17 00:00:00 2001 From: Nassim Boutekedjiret Date: Fri, 9 Oct 2026 20:15:02 +0200 Subject: [PATCH 7/7] Drop the quota subcode match: no host sends it Address live tests on Windows Server 2008 R2, 2016 and 2022: no host sends a QuotaLimit subcode with the quota fault. 2008 R2 sends WSManFault code 2150859174 with the generic InternalError subcode; 2016 and 2022 send no WSManFault code at all, only InternalError, which cannot identify this fault. So: - The quota fault is recognized by its code only, which means the retry-in-a-new-shell runs on 2008 R2 only. Replacing the shell every maxCommandsPerShell commands stays the main fix, the only one that works on later versions. - WinRMFaultException.getFaultSubcode(), added for that match, is removed (back to main), with its fault detail table row. - The builder Javadoc, commands.md, preparing-the-host.md and the CHANGELOG scope the quota retry to Windows Server 2008 R2. - Tests: the subcode test and its made-up fixture are gone; the quota tests use the fault as 2008 R2 sends it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +++-- .../org/metricshub/winrm/WinRMClient.java | 8 +-- .../winrm/exceptions/WinRMFaultException.java | 36 -------------- .../metricshub/winrm/light/WsmanClient.java | 29 ++--------- src/site/markdown/commands.md | 8 +-- src/site/markdown/preparing-the-host.md | 2 +- src/site/markdown/timeouts-and-errors.md | 1 - .../org/metricshub/winrm/WinRMClientTest.java | 49 +++++-------------- 8 files changed, 33 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c46dcff..c2a0e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,14 +164,13 @@ Consequences: with every command refused with WSManFault 2150859174 (*the maximum number of concurrent operations for this user has been exceeded*), after 15 commands on Windows Server 2008 R2 and 1500 on later versions, until it was recreated. The client now replaces its shell every 10 - commands (the new `WinRMClient.Builder.maxCommandsPerShell(int)` changes that number), retries - once in a new shell a command the quota refuses in a shell that already ran commands, and never + commands (the new `WinRMClient.Builder.maxCommandsPerShell(int)` changes that number) and never reuses a shell holding a command it could not terminate (its terminate `Signal` failed or was - skipped). The new shell gets the same working directory, environment and profile; deleting the - old one ends any process a previous command left running in it, as closing the client always - did. The quota fault is recognized by its code or, on hosts that send none (a French - Windows Server 2022), by its SOAP subcode, now exposed as - `WinRMFaultException.getFaultSubcode()`. + skipped). On Windows Server 2008 R2, whose quota fault carries its WSManFault code, a command + the quota refuses in a shell that already ran commands is also retried once in a new shell; + later versions send that fault with no code, so nothing reliable identifies it. The new shell + gets the same working directory, environment and profile; deleting the old one ends any process + a previous command left running in it, as closing the client always did. - **A streaming read resumed after a long pause no longer times out spuriously** (#198). When a streaming consumer (e.g. a `RemoteProcess` read slowly) paused longer than the inactivity diff --git a/src/main/java/org/metricshub/winrm/WinRMClient.java b/src/main/java/org/metricshub/winrm/WinRMClient.java index 91eea7c..7bbd2aa 100644 --- a/src/main/java/org/metricshub/winrm/WinRMClient.java +++ b/src/main/java/org/metricshub/winrm/WinRMClient.java @@ -637,9 +637,11 @@ public Builder arraySeparator(final String arraySeparator) { * command refused. Replacing the shell (one Delete and one Create, about 100 ms) releases * them. The new shell gets the same working directory, environment and profile, but * deleting the old one ends any process a previous command left running in it, as - * {@link WinRMClient#close()} does. Besides, a command the quota refuses in a shell that - * already ran commands is retried once in a new shell; in a fresh shell, which holds - * nothing to release, the fault is reported. + * {@link WinRMClient#close()} does. Besides, on Windows Server 2008 R2, whose quota fault + * carries its WSManFault code, a command the quota refuses in a shell that already ran + * commands is retried once in a new shell; in a fresh shell, which holds nothing to + * release, the fault is reported. Later versions send that fault with no code, so this + * setting is what keeps a client under the quota there. *

* A lower value leaves more of the quota to the user's other connections at the cost of * more frequent replacements; 1 runs every command in a shell of its own, like {@code winrs}. diff --git a/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java b/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java index 99e64c2..e72a00b 100644 --- a/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java +++ b/src/main/java/org/metricshub/winrm/exceptions/WinRMFaultException.java @@ -35,37 +35,15 @@ public class WinRMFaultException extends WinRMClientException { private final int httpStatus; private final String faultCode; - private final String faultSubcode; private final String faultReason; private final String faultDetail; - /** - * Create the exception, without a SOAP fault subcode. - * - * @param message the complete detail message (same format as the legacy API) - * @param httpStatus the HTTP status of the faulting response - * @param faultCode the WSManFault code, or {@code null} when the response carried none - * @param faultReason the SOAP fault reason text, or {@code null} - * @param faultDetail the detailed WSManFault message (provider-level detail), or {@code null} - */ - public WinRMFaultException( - final String message, - final int httpStatus, - final String faultCode, - final String faultReason, - final String faultDetail - ) { - this(message, httpStatus, faultCode, null, faultReason, faultDetail); - } - /** * Create the exception. * * @param message the complete detail message (same format as the legacy API) * @param httpStatus the HTTP status of the faulting response * @param faultCode the WSManFault code, or {@code null} when the response carried none - * @param faultSubcode the SOAP fault subcode without its namespace prefix (e.g. - * {@code QuotaLimit}), or {@code null} when the response carried none * @param faultReason the SOAP fault reason text, or {@code null} * @param faultDetail the detailed WSManFault message (provider-level detail), or {@code null} */ @@ -73,14 +51,12 @@ public WinRMFaultException( final String message, final int httpStatus, final String faultCode, - final String faultSubcode, final String faultReason, final String faultDetail ) { super(message); this.httpStatus = httpStatus; this.faultCode = faultCode; - this.faultSubcode = faultSubcode; this.faultReason = faultReason; this.faultDetail = faultDetail; } @@ -103,18 +79,6 @@ public String getFaultCode() { return faultCode; } - /** - * Get the SOAP fault subcode, without its namespace prefix: the WS-Management fault name, such - * as {@code QuotaLimit} or {@code InvalidSelectors}. Unlike the reason text it is never - * translated, and some hosts send it without any WSManFault code (e.g. the quota fault of a - * French Windows Server 2022). - * - * @return the fault subcode, or {@code null} when the response carried none - */ - public String getFaultSubcode() { - return faultSubcode; - } - /** * Get the SOAP fault reason text. * diff --git a/src/main/java/org/metricshub/winrm/light/WsmanClient.java b/src/main/java/org/metricshub/winrm/light/WsmanClient.java index 4ebf13f..867dae9 100644 --- a/src/main/java/org/metricshub/winrm/light/WsmanClient.java +++ b/src/main/java/org/metricshub/winrm/light/WsmanClient.java @@ -59,13 +59,10 @@ final class WsmanClient implements AutoCloseable { private static final String FAULT_OPERATION_TIMEOUT = "2150858793"; private static final String FAULT_SHELL_NOT_FOUND = "2150858843"; - // The user's MaxConcurrentOperationsPerUser quota is full. Some hosts (a French Windows Server - // 2022) send this fault without any WSManFault code, so it is also recognized by its SOAP - // subcode, which is never translated. + // The user's MaxConcurrentOperationsPerUser quota is full. Only Windows Server 2008 R2 sends this + // code: later versions send the fault with no WSManFault code and a generic InternalError + // subcode, which nothing reliable distinguishes from other faults (measured on 2016 and 2022). private static final String FAULT_OPERATION_QUOTA = "2150859174"; - private static final String SUBCODE_QUOTA_LIMIT = "QuotaLimit"; - - private static final String SOAP_ENVELOPE_NS = "http://www.w3.org/2003/05/soap-envelope"; // A Send answered with this Windows error (ERROR_NO_DATA, "The pipe is being closed") found the // command no longer reading its standard input: it exited, or closed its stdin, first. @@ -1219,7 +1216,6 @@ private static WinRMFaultException faultException(final String operation, final operation + " failed: " + faultSummary(resp), resp.status, trimToNull(wsmanFaultCode(resp.document)), - trimToNull(faultSubcode(resp.document)), trimToNull(text(resp.document, "Text")), trimToNull(wsmanFaultMessage(resp.document)) ); @@ -1491,26 +1487,9 @@ private static String wsmanFaultCode(final Document doc) { return faults.getLength() > 0 ? ((Element) faults.item(0)).getAttribute("Code") : null; } - /** - * The SOAP fault subcode without its namespace prefix (e.g. {@code QuotaLimit}), or null: the - * fault's name, never translated, which some hosts send without any WSManFault code. - */ - private static String faultSubcode(final Document doc) { - final NodeList subcodes = doc.getElementsByTagNameNS(SOAP_ENVELOPE_NS, "Subcode"); - if (subcodes.getLength() == 0) { - return null; - } - final NodeList values = ((Element) subcodes.item(0)).getElementsByTagNameNS(SOAP_ENVELOPE_NS, "Value"); - if (values.getLength() == 0) { - return null; - } - final String value = values.item(0).getTextContent().trim(); - return value.substring(value.indexOf(':') + 1); - } - /** Whether the fault says the user's WSMan operation quota (MaxConcurrentOperationsPerUser) is full. */ private static boolean isQuotaFault(final WinRMFaultException fault) { - return FAULT_OPERATION_QUOTA.equals(fault.getFaultCode()) || SUBCODE_QUOTA_LIMIT.equals(fault.getFaultSubcode()); + return FAULT_OPERATION_QUOTA.equals(fault.getFaultCode()); } /** diff --git a/src/site/markdown/commands.md b/src/site/markdown/commands.md index 1a79a34..bd7cfc4 100644 --- a/src/site/markdown/commands.md +++ b/src/site/markdown/commands.md @@ -98,9 +98,11 @@ So the client replaces its shell: * every 10 commands. The builder's `maxCommandsPerShell(int)` changes that number: lower leaves more of the quota to the user's other connections, and 1 runs every command in a shell of its own, like `winrs`; -* when the quota refuses a command in a shell that already ran commands: the command ran nothing, - so it is retried once in a new shell. In a fresh shell, which holds nothing to release, the fault - is reported; +* on Windows Server 2008 R2, when the quota refuses a command in a shell that already ran + commands: the command ran nothing, so it is retried once in a new shell. In a fresh shell, which + holds nothing to release, the fault is reported. Later versions send the quota fault without its + WSManFault code, so nothing reliable identifies it: there, replacing the shell every N commands + is what keeps the client under the quota; * when a command could not be terminated cleanly (its terminate `Signal` failed or was skipped). A replacement costs a Delete and a Create (about 100 ms). The new shell gets the same working diff --git a/src/site/markdown/preparing-the-host.md b/src/site/markdown/preparing-the-host.md index 2905aaf..90474e5 100644 --- a/src/site/markdown/preparing-the-host.md +++ b/src/site/markdown/preparing-the-host.md @@ -369,7 +369,7 @@ the host, the tighter they are: | --- | --- | --- | | `MaxMemoryPerShellMB` | Memory per shell, including child processes | Historically **150 MB**; 1024 MB on modern hosts. A command whose output is large can hit it. | | `MaxShellsPerUser` | Concurrent shells per user | 5 on older hosts, 30 on modern ones. Close clients you no longer need. | -| `MaxConcurrentOperationsPerUser` | Concurrent operations per user | 15 on Windows Server 2008 R2, 1500 later. File transfers are batched specifically to stay under low limits. Every command holds one until its shell is deleted, even after it completed: the client replaces its shell every 10 commands and when the quota refuses a command in a shell that already ran commands, so a long-lived client does not pile them up (see [Shell reuse](commands.html#shell-reuse)). | +| `MaxConcurrentOperationsPerUser` | Concurrent operations per user | 15 on Windows Server 2008 R2, 1500 later. File transfers are batched specifically to stay under low limits. Every command holds one until its shell is deleted, even after it completed: the client replaces its shell every 10 commands (and, on 2008 R2, when the quota refuses a command in a shell that already ran commands), so a long-lived client does not pile them up (see [Shell reuse](commands.html#shell-reuse)). | | `MaxEnvelopeSizekb` | SOAP envelope size | 150 KB on older hosts, **500 KB** on modern ones. This client always uses 150 KB envelopes, so the default never needs raising. | | `IdleTimeout` | How long an idle shell survives | 180000 ms (3 min) on older hosts, **7200000 ms** (2 h) on modern ones; 60000 ms minimum. The client transparently recreates a shell the host reaped. | diff --git a/src/site/markdown/timeouts-and-errors.md b/src/site/markdown/timeouts-and-errors.md index 789403a..539b8bb 100644 --- a/src/site/markdown/timeouts-and-errors.md +++ b/src/site/markdown/timeouts-and-errors.md @@ -115,7 +115,6 @@ the fault **programmatically**, so no message parsing is needed: | Method | Returns | | --- | --- | | `getFaultCode()` | The WSManFault code as a decimal `String`, e.g. `"2150858778"`, or `null` when the response carried none. | -| `getFaultSubcode()` | The SOAP fault subcode without its prefix, e.g. `"QuotaLimit"`, or `null`. Never translated, and present on hosts that send no WSManFault code (the quota fault of a French Windows Server 2022). | | `getFaultReason()` | The SOAP fault reason text, or `null`. | | `getFaultDetail()` | The provider-level detail — where WMI puts mnemonics such as `WBEM_E_INVALID_CLASS` or `WBEM_E_INVALID_NAMESPACE` — or `null`. | | `getHttpStatus()` | The HTTP status of the faulting response (typically 500). | diff --git a/src/test/java/org/metricshub/winrm/WinRMClientTest.java b/src/test/java/org/metricshub/winrm/WinRMClientTest.java index 6fed5a1..b89ff5c 100644 --- a/src/test/java/org/metricshub/winrm/WinRMClientTest.java +++ b/src/test/java/org/metricshub/winrm/WinRMClientTest.java @@ -23,7 +23,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.metricshub.winrm.light.FakeWsmanResponses.commandResponse; @@ -69,19 +68,11 @@ class WinRMClientTest { private static final String WSEN = "http://schemas.xmlsoap.org/ws/2004/09/enumeration"; private static final String WSMAN = "http://schemas.dmtf.org/wbem/wsman/1/wsman.xsd"; - /** - * What a French Windows Server 2022 answers when the user's operation quota is full: a SOAP - * fault with the {@code QuotaLimit} subcode and a translated reason, but no WSManFault element. - */ - private static final String QUOTA_FAULT_WITHOUT_CODE = "s:Receiver" + - "w:QuotaLimit" + - "Le nombre maximal d'opérations simultanées pour cet utilisateur a été dépassé." - + - ""; + /** The quota fault, as Windows Server 2008 R2 sends it: the only version that carries its code. */ + private static final String QUOTA_FAULT = fault( + "2150859174", + "The WS-Management service cannot process the request. The maximum number of concurrent operations for this user has been exceeded." + ); private FakeWsmanServer server; @@ -1003,24 +994,10 @@ void theShellIsReplacedAfterTenCommandsByDefault() throws Exception { @Test void aCommandRefusedByTheQuotaIsRetriedInANewShell() throws Exception { - assertQuotaFaultReplacesTheShell( - fault( - "2150859174", - "The WS-Management service cannot process the request. The maximum number of concurrent operations for this user has been exceeded." - ) - ); - } - - @Test - void aQuotaFaultWithoutWsmanFaultCodeIsRecognizedByItsSubcode() throws Exception { - assertQuotaFaultReplacesTheShell(QUOTA_FAULT_WITHOUT_CODE); - } - - private void assertQuotaFaultReplacesTheShell(final String quotaFault) throws Exception { server.enqueue(200, envelope(resourceCreated("SHELL-1"))); enqueueCommand("CMD-1", "first"); // The user's operation quota is full: the second Command is refused before it could run. - server.enqueue(500, quotaFault); + server.enqueue(500, QUOTA_FAULT); enqueueShellReplacement("SHELL-2"); enqueueCommand("CMD-2", "second"); @@ -1038,15 +1015,13 @@ private void assertQuotaFaultReplacesTheShell(final String quotaFault) throws Ex } @Test - void aQuotaFaultOnAFreshShellIsReportedWithItsSubcode() throws Exception { + void aQuotaFaultOnAFreshShellIsReported() throws Exception { // The new shell ran nothing: deleting it would release nothing, so the fault is reported. - server.enqueue(200, envelope(resourceCreated("SHELL-1"))).enqueue(500, QUOTA_FAULT_WITHOUT_CODE); + server.enqueue(200, envelope(resourceCreated("SHELL-1"))).enqueue(500, QUOTA_FAULT); try (WinRMClient client = builder(PASSWORD).build()) { final WinRMFaultException e = assertThrows(WinRMFaultException.class, () -> client.command("x.exe").execute()); - // No WSManFault code on such hosts: the untranslated subcode still names the fault. - assertNull(e.getFaultCode()); - assertEquals("QuotaLimit", e.getFaultSubcode()); + assertEquals("2150859174", e.getFaultCode()); assertEquals(2, server.decryptedRequests().size(), "no Delete, no retry"); } } @@ -1057,9 +1032,9 @@ void aCommandRefusedByTheQuotaIsRetriedOnlyOnce() throws Exception { // the fault is reported instead of looping. server.enqueue(200, envelope(resourceCreated("SHELL-1"))); enqueueCommand("CMD-1", "first"); - server.enqueue(500, QUOTA_FAULT_WITHOUT_CODE); + server.enqueue(500, QUOTA_FAULT); enqueueShellReplacement("SHELL-2"); - server.enqueue(500, QUOTA_FAULT_WITHOUT_CODE); + server.enqueue(500, QUOTA_FAULT); try (WinRMClient client = builder(PASSWORD).build()) { assertEquals("first", client.command("first.exe").execute().stdout()); @@ -1067,7 +1042,7 @@ void aCommandRefusedByTheQuotaIsRetriedOnlyOnce() throws Exception { WinRMFaultException.class, () -> client.command("second.exe").execute() ); - assertEquals("QuotaLimit", e.getFaultSubcode()); + assertEquals("2150859174", e.getFaultCode()); final long commands = server.decryptedRequests().stream().filter(r -> r.contains(":CommandLine>")).count(); assertEquals(3, commands, "the first command, the refused one, and a single retry"); }