From bc9693dbe50ca61fe1ac7b0808d222bd7f0a9f1b Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 24 Sep 2026 13:04:49 +0200 Subject: [PATCH 1/2] Read remote files: client.file(path) with byte ranges, streaming and digests (#146) Foundation of the remote file access family (#146, then #145, #147, #148): client.file(path) returns a RemoteFile request with readBytes(), readText(Charset), openStream(), openReader(Charset) and digest(String), plus offset(long) (negative = from the end), length(long), maxBytes(long) (64 MiB default cap) and timeout(Duration). The package-private RemoteFiles primitive runs a small PowerShell script (-EncodedCommand, path embedded base64-encoded, no quoting) that opens the file with a ReadWrite share mode, seeks, and writes each block as one base64 line to the raw stdout stream; the client decodes the lines as they arrive and maps the script's exit code to explicit failures (not found, directory, access denied, sharing violation, Constrained Language Mode, PowerShell missing). Blocks are 49,149 bytes so each 65,533-byte line fills exactly two of the WinRM service's 32 KiB pipe reads (at most ~60 per second per stream): 20 MiB now reads in 14-15 s on Windows Server 2008 R2, 2019 and 2022, instead of 101 s with Console.Out-written 76 KB lines. Tests: the scripts run in the local powershell.exe on Windows (RemoteFilesScriptTest), client-side decoding against FakeWsmanServer (RemoteFileTest), and live tests; docs in the new files.md page. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 + .../java/org/metricshub/winrm/RemoteFile.java | 307 +++++++++++++++ .../org/metricshub/winrm/RemoteFiles.java | 349 ++++++++++++++++++ .../org/metricshub/winrm/WinRMClient.java | 19 + src/site/markdown/file-transfers.md | 1 + src/site/markdown/files.md | 123 ++++++ src/site/markdown/index.md | 1 + src/site/site.xml | 1 + .../org/metricshub/winrm/RemoteFileTest.java | 258 +++++++++++++ .../winrm/RemoteFilesScriptTest.java | 196 ++++++++++ .../org/metricshub/winrm/WinRMLiveTest.java | 80 ++++ 11 files changed, 1339 insertions(+) create mode 100644 src/main/java/org/metricshub/winrm/RemoteFile.java create mode 100644 src/main/java/org/metricshub/winrm/RemoteFiles.java create mode 100644 src/site/markdown/files.md create mode 100644 src/test/java/org/metricshub/winrm/RemoteFileTest.java create mode 100644 src/test/java/org/metricshub/winrm/RemoteFilesScriptTest.java diff --git a/README.md b/README.md index a40ab5d..e23dc5d 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ through the unchecked `WinRMClientException` hierarchy (`WinRMAuthenticationExce `WqlSyntaxException`). ```java +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Duration; import org.metricshub.winrm.*; @@ -98,6 +99,9 @@ try (WinRMClient client = WinRMClient.builder("server01.acme.com") // Copy a file to the host (through the WinRM channel itself — no SMB) client.uploadFile(Path.of("collect.ps1"), "C:\\Windows\\Temp\\collect.ps1"); + + // Read a remote file — whole, a byte range, or the tail of a log + String tail = client.file("D:\\logs\\app.log").offset(-8192).readText(StandardCharsets.UTF_8); } ``` diff --git a/src/main/java/org/metricshub/winrm/RemoteFile.java b/src/main/java/org/metricshub/winrm/RemoteFile.java new file mode 100644 index 0000000..bfa79fb --- /dev/null +++ b/src/main/java/org/metricshub/winrm/RemoteFile.java @@ -0,0 +1,307 @@ +package org.metricshub.winrm; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * WinRM Java Client + * ჻჻჻჻჻჻ + * Copyright (C) 2023 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import org.metricshub.winrm.exceptions.WinRMClientException; +import org.metricshub.winrm.exceptions.WinRMTimeoutException; + +/** + * A file on the remote host, obtained with {@link WinRMClient#file(String)}: read its content — + * whole, as a byte range, or as a stream — or compute its digest. Nothing is sent until a + * terminal ({@link #readBytes()}, {@link #readText(Charset)}, {@link #openStream()}, + * {@link #openReader(Charset)}, {@link #digest(String)}) is called. + * + * 
{@code
+ * byte[] content = client.file("C:\\Windows\\Temp\\collect.bin").readBytes();
+ *
+ * // The last 8 KiB of a log: one seek, no scan
+ * String tail = client.file("D:\\logs\\huge.log").offset(-8192).readText(StandardCharsets.UTF_8);
+ *
+ * try (BufferedReader reader = client.file("D:\\logs\\huge.log").openReader(StandardCharsets.UTF_8)) {
+ * 	reader.lines().filter(l -> l.contains("ERROR")).forEach(System.out::println);
+ * }
+ * }
+ *

+ * The content travels through the WinRM connection itself (no SMB, no extra port): a small + * PowerShell script on the host writes the bytes base64-encoded, which is binary-safe and + * independent of the remote console code page. It is designed for configuration files, logs and + * small data files — not a bulk transport: base64 through a command shell is far slower + * than SMB. The host needs PowerShell (2.0 or later) in {@code FullLanguage} mode. + *

+ * The file is opened with a share mode that tolerates other writers, so a log being written by a + * running service can be read; a file held with an exclusive lock (e.g. {@code pagefile.sys}) + * fails with a sharing violation. Ranges are byte ranges, and reads are not snapshots: a + * file that grows or shrinks between two reads is read as it is at each read. + *

+ * A request is not thread-safe; configure it and call its terminals from one thread. + */ +public final class RemoteFile { + + /** Default cap of {@link #readBytes()} and {@link #readText(Charset)}: 64 MiB. */ + public static final long DEFAULT_MAX_BYTES = 64L * 1024 * 1024; + + /** The largest byte array the JVM reliably allocates. */ + private static final long MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8L; + + private final WinRMClient client; + private final String path; + private long offset; + private long length = -1; + private long maxBytes = DEFAULT_MAX_BYTES; + private Duration timeout; + + /** + * Create the request. + * + * @param client the client the file is read through + * @param path the path of the file on the remote host + */ + RemoteFile(final WinRMClient client, final String path) { + Utils.checkNonBlank(path, "path"); + this.client = client; + this.path = path; + this.timeout = client.defaultTimeout(); + } + + /** + * Get the path of the file on the remote host. + * + * @return the path, as given to {@link WinRMClient#file(String)} + */ + public String path() { + return path; + } + + /** + * Start reading at the given byte position instead of the beginning of the file. A + * negative offset counts from the end: {@code offset(-n)} starts at + * {@code max(0, size - n)}, the size being read by the same remote invocation that seeks (so a + * growing log is tailed from its current end); a file shorter than {@code n} is read whole. An + * offset past the end of the file reads nothing — it is not an error. Applies to every read + * terminal, not to {@link #digest(String)}. + * + * @param offset the byte position; negative counts from the end of the file + * @return this request + */ + public RemoteFile offset(final long offset) { + this.offset = offset; + return this; + } + + /** + * Read at most the given number of bytes (from the {@link #offset(long)}, once resolved) + * instead of reading to the end of the file. A length beyond the end of the file reads what + * exists; a zero length reads nothing. Applies to every read terminal, not to + * {@link #digest(String)}. + * + * @param length how many bytes to read at most (zero or more) + * @return this request + * @throws IllegalArgumentException when the length is negative + */ + public RemoteFile length(final long length) { + if (length < 0) { + throw new IllegalArgumentException("length must not be negative."); + } + this.length = length; + return this; + } + + /** + * Set the size cap of {@link #readBytes()} and {@link #readText(Charset)}, which hold the whole + * content in memory: reading more than the cap fails instead of risking an + * {@link OutOfMemoryError}. Defaults to {@link #DEFAULT_MAX_BYTES} (64 MiB). The cap is also + * sent to the host, so no more than one byte past it is ever transferred. For larger content, + * use {@link #openStream()}, which reads with bounded memory and has no cap. + * + * @param maxBytes the largest content accepted, in bytes + * @return this request + * @throws IllegalArgumentException when the cap is negative or exceeds the largest possible + * byte array + */ + public RemoteFile maxBytes(final long maxBytes) { + if (maxBytes < 0 || maxBytes >= MAX_ARRAY_SIZE) { + throw new IllegalArgumentException("maxBytes must be between 0 and " + (MAX_ARRAY_SIZE - 1) + "."); + } + this.maxBytes = maxBytes; + return this; + } + + /** + * Override the client's timeout for this request. For the blocking terminals + * ({@link #readBytes()}, {@link #readText(Charset)}, {@link #digest(String)}) it is a + * wall-clock deadline for the whole read; for {@link #openStream()} and + * {@link #openReader(Charset)} it is an inactivity timeout, the longest silence + * tolerated from the host between two chunks of content. + * + * @param timeout the timeout (at least one millisecond) + * @return this request + */ + public RemoteFile timeout(final Duration timeout) { + this.timeout = WinRMClient.checkPositive(timeout, "timeout"); + return this; + } + + /** + * Read the content (the whole file, or the configured range) into memory, byte-exact: nothing + * is converted, and a byte order mark is kept. + * + * @return the content + * @throws WinRMClientException when the content exceeds {@link #maxBytes(long)}, or the file + * cannot be read (not found, a directory, access denied, sharing violation, PowerShell + * unavailable or constrained) + * @throws WinRMTimeoutException when the timeout elapses first + */ + public byte[] readBytes() { + final long requested = length < 0 ? maxBytes + 1 : Math.min(length, maxBytes + 1); + return blocking(() -> { + try (InputStream in = open(requested)) { + final byte[] content = in.readNBytes((int) requested); + if (content.length > maxBytes) { + throw new WinRMClientException( + String.format( + "Remote file %s on %s exceeds the %d-byte cap of readBytes(): raise maxBytes(...), read a range, or use openStream()", + path, + client.hostname(), + maxBytes + ) + ); + } + return content; + } + }); + } + + /** + * Read the content (the whole file, or the configured range) into memory and decode it with the + * given charset — no guessing, no default. A byte order mark is not stripped (it + * decodes to {@code U+FEFF}). A range boundary may split a multibyte character, which then + * decodes to {@code U+FFFD} at the edges, like any malformed input. + * + * @param charset the charset the file is encoded with + * @return the decoded content + * @throws WinRMClientException when the content exceeds {@link #maxBytes(long)}, or the file + * cannot be read + * @throws WinRMTimeoutException when the timeout elapses first + */ + public String readText(final Charset charset) { + Utils.checkNonNull(charset, "charset"); + return new String(readBytes(), charset); + } + + /** + * Open the content (the whole file, or the configured range) as a stream, decoded as it + * arrives: memory is bounded by one transfer block, not by the file, and there is no size cap. + * The file is opened before this method returns, so a file that cannot be read fails here; a + * failure midway is reported by {@code read()} (never as a silently short read). + *

+ * The stream must be closed — use try-with-resources. It holds the client's connection + * until it reaches its end or is closed; closing it early stops the remote read. Failures are + * reported through the unchecked {@link WinRMClientException} hierarchy, including from + * {@code read()}. The timeout is an inactivity timeout (see {@link #timeout(Duration)}). + * + * @return the content stream, to use with try-with-resources + * @throws WinRMClientException when the file cannot be read + * @throws WinRMTimeoutException when the host does not answer in time + */ + public InputStream openStream() { + return open(length); + } + + /** + * Open the content (the whole file, or the configured range) as a character stream decoded + * with the given charset: {@link #openStream()} behind an {@link InputStreamReader}, with the + * same lifecycle — close it. A byte order mark is not stripped. + * + * @param charset the charset the file is encoded with + * @return the reader, to use with try-with-resources + * @throws WinRMClientException when the file cannot be read + * @throws WinRMTimeoutException when the host does not answer in time + */ + public BufferedReader openReader(final Charset charset) { + Utils.checkNonNull(charset, "charset"); + return new BufferedReader(new InputStreamReader(openStream(), charset)); + } + + /** + * Compute the digest of the whole file on the host — nothing but the digest is transferred. + * The {@link #offset(long)} and {@link #length(long)} settings do not apply. + * + * @param algorithm {@code MD5}, {@code SHA1}, {@code SHA256}, {@code SHA384} or {@code SHA512} + * @return the digest, as lowercase hexadecimal + * @throws IllegalArgumentException when the algorithm is not supported + * @throws WinRMClientException when the file cannot be read + * @throws WinRMTimeoutException when the timeout elapses first + */ + public String digest(final String algorithm) { + Utils.checkNonNull(algorithm, "algorithm"); + final String script = RemoteFiles.digestScript(path, algorithm); + return blocking(() -> { + try (InputStream in = RemoteFiles.open(client, path, script, timeout)) { + final StringBuilder hex = new StringBuilder(); + for (final byte b : in.readAllBytes()) { + hex.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); + } + return hex.toString(); + } + }); + } + + /** Start the remote read of the configured offset and the given length ({@code -1}: to the end). */ + private InputStream open(final long readLength) { + return RemoteFiles.open(client, path, RemoteFiles.readScript(path, offset, readLength), timeout); + } + + /** + * Run a blocking terminal under the wall-clock deadline: a worker runs the exchange and is + * cancelled when the deadline fires, exactly like {@link CommandRequest#execute()}. + */ + private T blocking(final Callable task) { + try { + return Utils.execute(task, WinRMClient.toMillis(timeout)); + } catch (final TimeoutException e) { + throw new WinRMTimeoutException( + String.format("Reading remote file %s timed out after %s on %s", path, timeout, client.hostname()), + e + ); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new WinRMClientException(e.getMessage(), e); + } catch (final ExecutionException e) { + final Throwable cause = e.getCause() != null ? e.getCause() : e; + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + if (cause instanceof IOException) { + throw new WinRMClientException(cause.getMessage(), cause); + } + throw new WinRMClientException(String.valueOf(cause.getMessage()), cause); + } + } +} diff --git a/src/main/java/org/metricshub/winrm/RemoteFiles.java b/src/main/java/org/metricshub/winrm/RemoteFiles.java new file mode 100644 index 0000000..e75ec93 --- /dev/null +++ b/src/main/java/org/metricshub/winrm/RemoteFiles.java @@ -0,0 +1,349 @@ +package org.metricshub.winrm; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * WinRM Java Client + * ჻჻჻჻჻჻ + * Copyright (C) 2023 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.metricshub.winrm.exceptions.WinRMClientException; + +/** + * The remote file access primitive: small PowerShell scripts that open a remote file and write + * what the caller asked for as base64 lines on stdout, and the {@link InputStream} that + * decodes those lines as the output chunks arrive. + *

+ * Base64 is what makes a text console a binary-safe channel: every byte value survives, and the + * output is pure ASCII, so recovering the exact bytes never depends on the remote console code + * page. Each line is an independent base64 block, decoded on its own. The caller-supplied path is + * embedded base64-encoded too (UTF-8), so no path ever needs quoting or escaping. + *

+ * Failures are reported by the script's exit code, checked when the output ends (see the + * {@code EXIT_*} constants), and turned into a {@link WinRMClientException} naming the cause. + */ +final class RemoteFiles { + + private RemoteFiles() {} + + /** The path does not exist. */ + static final int EXIT_NOT_FOUND = 2; + + /** The file is held with an exclusive lock by another process (sharing violation). */ + static final int EXIT_SHARING_VIOLATION = 3; + + /** Access to the path is denied. */ + static final int EXIT_ACCESS_DENIED = 4; + + /** PowerShell runs in Constrained Language Mode: the .NET calls the scripts rely on are blocked. */ + static final int EXIT_CONSTRAINED_LANGUAGE = 5; + + /** The path is a directory, not a file. */ + static final int EXIT_IS_DIRECTORY = 6; + + /** The exit code of {@code cmd.exe} when {@code powershell.exe} cannot be found. */ + static final int EXIT_COMMAND_NOT_FOUND = 9009; + + /** + * Bytes read and written per base64 line: 49,149, a multiple of 3 so a full block encodes + * without padding, into 65,532 characters — with the newline, a 65,533-byte line that fits in + * exactly two of the WinRM service's output reads. The service's shell plugin reads a + * command's stdout pipe with at most one 32 KiB read per timer tick (64 per second at the + * default 15.625 ms), so each line should fill whole reads: a 57 KiB block (a 77,825-byte + * line) needs three, and is measurably slower. + */ + static final int BLOCK_SIZE = 49_149; + + /** + * The hash algorithms {@link #digestScript(String, String)} accepts: the names .NET's + * {@code HashAlgorithm.Create} knows on every Windows version (PowerShell 2.0 included). The + * name is embedded in the script, hence the closed list. + */ + static final Set DIGEST_ALGORITHMS = Set.of("MD5", "SHA1", "SHA256", "SHA384", "SHA512"); + + /** + * The opening shared by every script: check the language mode, define {@code fail}, which maps + * a .NET failure to the documented exit code (a sharing violation is HRESULT + * {@code 0x80070020}, 32; a byte-range lock violation {@code 0x80070021}, 33), then open the + * file with a share mode that tolerates other writers ({@code ReadWrite}) — a log written by a + * running service is the most common thing to read, and {@code File.OpenRead} fails on it. + * {@code exit} in a function exits the whole script. {@code %s} is the base64 of the path's + * UTF-8 bytes. + */ + private static final String OPEN_FILE = "if($ExecutionContext.SessionState.LanguageMode -ne 'FullLanguage'){exit 5};" + + + "$ErrorActionPreference='Stop';" + + "function fail($e){if($e.InnerException){$e=$e.InnerException};" + + "[Console]::Error.WriteLine($e.Message);" + + "if($e -is [IO.FileNotFoundException] -or $e -is [IO.DirectoryNotFoundException]){exit 2};" + + "if($e -is [UnauthorizedAccessException]){exit 4};" + + "$h=[Runtime.InteropServices.Marshal]::GetHRForException($e) -band 0xFFFF;" + + "if($h -eq 32 -or $h -eq 33){exit 3};" + + "exit 1};" + + "$p=[Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('%s'));" + + "if([IO.Directory]::Exists($p)){exit 6};" + + "try{$f=[IO.File]::Open($p,'Open','Read','ReadWrite')}catch{fail $_.Exception};"; + + /** + * Read a byte range: resolve a negative offset from the size of the open stream (so a + * growing log is tailed from its current end), seek, and write at most {@code length} bytes + * ({@code -1}: to the end) as one base64 line per block. Seeking past the end is legal and + * reads nothing. {@code %d} are the offset, the length and the block size. + *

+ * Each line goes to the raw stdout stream as ONE write: the WinRM service reads the pipe at + * most once per timer tick, taking only what is in the pipe, and {@code [Console]::Out} (an + * auto-flushing writer with a small buffer) cuts a line into 256-byte writes that leave each + * read with about 4 KiB — 5 to 6 times slower (measured on Windows 2008 R2 and 2022). + */ + private static final String READ_RANGE = "try{$n=[long]%d;$l=[long]%d;" + + "if($n -lt 0){$n=[Math]::Max([long]0,$f.Length+$n)};" + + "$f.Position=$n;$b=New-Object byte[] %d;$o=[Console]::OpenStandardOutput();" + + "while($l -ne 0){$c=$b.Length;if($l -gt 0 -and $l -lt $c){$c=[int]$l};" + + "$r=$f.Read($b,0,$c);if($r -le 0){break};" + + "$a=[Text.Encoding]::ASCII.GetBytes([Convert]::ToBase64String($b,0,$r)+\"`n\");" + + "$o.Write($a,0,$a.Length);if($l -gt 0){$l-=$r}};" + + "$o.Flush()}catch{fail $_.Exception}finally{$f.Close()}"; + + /** Hash the whole file and write the raw digest as one base64 line. {@code %s} is the algorithm. */ + private static final String DIGEST = "try{$h=[Security.Cryptography.HashAlgorithm]::Create('%s');" + + "[Console]::Out.WriteLine([Convert]::ToBase64String($h.ComputeHash($f)))}catch{fail $_.Exception}finally{$f.Close()}"; + + /** + * Build the script writing a byte range of the file. + * + * @param path the remote file + * @param offset the byte position to start from; negative counts from the end of the file + * @param length how many bytes to read at most, or {@code -1} to read to the end + * @return the PowerShell script + */ + static String readScript(final String path, final long offset, final long length) { + return openFile(path) + String.format(READ_RANGE, offset, length, BLOCK_SIZE); + } + + /** + * Build the script writing the digest of the file. + * + * @param path the remote file + * @param algorithm one of {@link #DIGEST_ALGORITHMS} + * @return the PowerShell script + */ + static String digestScript(final String path, final String algorithm) { + if (!DIGEST_ALGORITHMS.contains(algorithm)) { + throw new IllegalArgumentException( + "Unsupported digest algorithm: " + algorithm + " (supported: " + + DIGEST_ALGORITHMS.stream().sorted().collect(Collectors.joining(", ")) + ")" + ); + } + return openFile(path) + String.format(DIGEST, algorithm); + } + + private static String openFile(final String path) { + return String.format(OPEN_FILE, Base64.getEncoder().encodeToString(path.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Start the script and return the stream of the bytes it writes. The first line is fetched + * before returning, so a file that cannot be opened fails here, not on the first read. + * + * @param client the client to run the script on + * @param path the remote file, for the error messages + * @param script the script, from {@link #readScript} or {@link #digestScript} + * @param timeout the inactivity timeout of the stream + * @return the decoded stream; it must be closed + */ + static InputStream open(final WinRMClient client, final String path, final String script, final Duration timeout) { + final RemoteProcess process = client.powerShell(script).timeout(timeout).start(); + try { + return new DecodingStream(process, path, client.hostname()); + } catch (final RuntimeException e) { + process.close(); + throw e; + } + } + + /** + * Build the exception reporting a non-zero exit code of a script. + * + * @param exitCode the script's exit code + * @param path the remote file + * @param hostname the remote host + * @param stderr what the script wrote on stderr + * @return the exception to throw + */ + static WinRMClientException failure( + final int exitCode, + final String path, + final String hostname, + final String stderr + ) { + switch (exitCode) { + case EXIT_NOT_FOUND: + return new WinRMClientException(String.format("Remote file not found on %s: %s", hostname, path)); + case EXIT_SHARING_VIOLATION: + return new WinRMClientException( + String.format( + "Remote file %s on %s is locked by another process (sharing violation): it cannot be read", + path, + hostname + ) + ); + case EXIT_ACCESS_DENIED: + return new WinRMClientException(String.format("Access denied to remote file %s on %s", path, hostname)); + case EXIT_CONSTRAINED_LANGUAGE: + return new WinRMClientException( + String.format( + "PowerShell runs in Constrained Language Mode on %s (e.g. AppLocker or WDAC): remote file access needs FullLanguage", + hostname + ) + ); + case EXIT_IS_DIRECTORY: + return new WinRMClientException(String.format("Remote path %s on %s is a directory, not a file", path, hostname)); + case EXIT_COMMAND_NOT_FOUND: + return new WinRMClientException( + String.format("PowerShell is not available on %s: remote file access needs powershell.exe", hostname) + ); + default: + return new WinRMClientException( + String.format( + "Failed to read remote file %s on %s (exit code %d)%s", + path, + hostname, + exitCode, + stderr.isBlank() ? "" : ": " + stderr.strip() + ) + ); + } + } + + /** + * The bytes a script writes, decoded line by line as the output arrives: memory is bounded by + * one line, not by the file. The exit code is checked when the output ends, so a failure + * midway is reported instead of looking like a short file. + */ + private static final class DecodingStream extends InputStream { + + private final RemoteProcess process; + private final BufferedReader stdout; + private final String path; + private final String hostname; + private byte[] block = new byte[0]; + private int position; + private boolean ended; + + private DecodingStream(final RemoteProcess process, final String path, final String hostname) { + this.process = process; + this.stdout = process.stdout(); + this.path = path; + this.hostname = hostname; + fill(); + } + + @Override + public int read() { + return fill() ? block[position++] & 0xFF : -1; + } + + @Override + public int read(final byte[] buffer, final int offset, final int length) { + Objects.checkFromIndexSize(offset, length, buffer.length); + if (length == 0) { + return 0; + } + if (!fill()) { + return -1; + } + final int count = Math.min(length, block.length - position); + System.arraycopy(block, position, buffer, offset, count); + position += count; + return count; + } + + @Override + public int available() { + return block.length - position; + } + + /** Make sure unread bytes are buffered: {@code false} at the end of a successful output. */ + private boolean fill() { + while (position == block.length) { + if (ended) { + return false; + } + final String line = readLine(); + if (line == null) { + end(); + return false; + } + final String data = line.strip(); + if (data.isEmpty()) { + continue; + } + try { + block = Base64.getDecoder().decode(data); + } catch (final IllegalArgumentException e) { + throw new WinRMClientException( + String.format( + "Unexpected output while reading remote file %s on %s: %s", + path, + hostname, + data.length() > 80 ? data.substring(0, 80) + "..." : data + ), + e + ); + } + position = 0; + } + return true; + } + + private String readLine() { + try { + return stdout.readLine(); + } catch (final IOException e) { + // Unreachable: the reader reports failures unchecked (see RemoteProcess). + throw new WinRMClientException(e.getMessage(), e); + } + } + + /** The output ended: collect the exit code, release the connection, report a failure. */ + private void end() { + ended = true; + final int exitCode = process.waitFor(); + final String stderr = exitCode == 0 ? "" : process.stderr().lines().collect(Collectors.joining("\n")); + process.close(); + if (exitCode != 0) { + throw failure(exitCode, path, hostname, stderr); + } + } + + @Override + public void close() { + ended = true; + block = new byte[0]; + position = 0; + process.close(); + } + } +} diff --git a/src/main/java/org/metricshub/winrm/WinRMClient.java b/src/main/java/org/metricshub/winrm/WinRMClient.java index 159ffbb..8503f1d 100644 --- a/src/main/java/org/metricshub/winrm/WinRMClient.java +++ b/src/main/java/org/metricshub/winrm/WinRMClient.java @@ -170,6 +170,25 @@ public CommandRequest powerShell(final String script) { return new CommandRequest(this, script, true); } + /** + * Prepare access to a file on the remote host: read its content (whole, as a byte range, or as + * a stream) or compute its digest, through the WinRM connection itself. Nothing is sent until + * a terminal of the returned {@link RemoteFile} is called. + * + * 

{@code
+	 * byte[] content = client.file("C:\\Windows\\Temp\\collect.bin").readBytes();
+	 * String tail = client.file("D:\\logs\\huge.log").offset(-8192).readText(StandardCharsets.UTF_8);
+	 * }
+ * + * @param path the absolute path of the file on the remote host, e.g. + * {@code C:\Windows\Temp\collect.log} + * @return the remote file, to configure and read + * @throws IllegalArgumentException when the path is blank + */ + public RemoteFile file(final String path) { + return new RemoteFile(this, path); + } + /** * Copy a local file to an explicit path on the remote host, through the WinRM connection * itself (no SMB, no extra port). The transfer is digest-verified and skipped when the diff --git a/src/site/markdown/file-transfers.md b/src/site/markdown/file-transfers.md index 9e0a7b8..10bba1d 100644 --- a/src/site/markdown/file-transfers.md +++ b/src/site/markdown/file-transfers.md @@ -177,5 +177,6 @@ Notes: ## See also +* [Remote Files](files.html) — the other direction: reading remote files through the WinRM channel * [Remote Commands](commands.html) — the command builder that carries the transfer * [Preparing the Windows Host](preparing-the-host.html) — the privileges a transfer needs diff --git a/src/site/markdown/files.md b/src/site/markdown/files.md new file mode 100644 index 0000000..75d50b9 --- /dev/null +++ b/src/site/markdown/files.md @@ -0,0 +1,123 @@ +keywords: remote file, read file, byte range, tail, digest, base64, powershell, stream +description: How the WinRM Java Client reads files on the remote host through the WinRM channel itself — whole files, byte ranges, tails, streams, and digests. + +# Remote Files + + + +WinRM has no file-access operation of its own (nothing like SFTP's `READ`), so the client reads +remote files **through the WinRM command shell**: a small PowerShell script opens the file on the +host and writes the requested bytes base64-encoded, and the client decodes them as they arrive. +No SMB, no extra port, no share. This is the reverse direction of +[File Transfers](file-transfers.html). + +## Reading a file + +[`client.file(path)`](apidocs/org/metricshub/winrm/RemoteFile.html) designates a file on the +remote host; nothing is sent until a terminal is called: + +```java +// Whole file, bytes (byte-exact: nothing converted, a BOM is kept) +byte[] content = client.file("C:\\Windows\\Temp\\collect.bin").readBytes(); + +// Whole file, text — the charset is always explicit, never guessed +String text = client.file("C:\\inetpub\\logs\\u_ex260729.log").readText(StandardCharsets.UTF_8); + +// Integrity: only the digest is transferred (MD5, SHA1, SHA256, SHA384, SHA512) +String sha256 = client.file("C:\\Windows\\Temp\\collect.ps1").digest("SHA256"); +``` + +## Byte ranges and tails + +`offset(long)` and `length(long)` select a byte range, served by a **seek** on the host — reading +the end of a multi-gigabyte log costs the same as reading the start of a small one: + +```java +// 64 KiB from the 1 GiB position +byte[] chunk = client.file("D:\\logs\\huge.log").offset(1_073_741_824L).length(65_536).readBytes(); + +// The last 8 KiB — a negative offset counts from the end +byte[] tail = client.file("D:\\logs\\huge.log").offset(-8192).readBytes(); +``` + +| Setting | Result | +|---|---| +| `offset(-n)` | starts at `max(0, size - n)`: a file shorter than `n` is read whole. The size is read by the same remote invocation that seeks, so a growing log is tailed from its current end. | +| `offset(-8192).length(1024)` | the first 1 KiB of the last 8 KiB | +| offset past the end | empty — not an error | +| `length` beyond the end | what exists | +| `length(0)`, an empty file | empty | + +Ranges are **byte** ranges: a range boundary can split a multibyte character, which then decodes +to `U+FFFD` at the edges of a `readText`. Reads are not snapshots: a file that changes between two +reads is read as it is at each read. + +## Streaming large files + +`readBytes()` and `readText()` hold the whole content in memory, so they are capped: +**64 MiB by default**, adjustable with `maxBytes(long)` — reading more fails with a clear +exception instead of an `OutOfMemoryError` (the cap is sent to the host too, so no more than one +byte past it is transferred). For anything larger, stream: + +```java +try (InputStream in = client.file("D:\\logs\\huge.log").openStream()) { + in.transferTo(out); +} + +try (BufferedReader reader = client.file("D:\\logs\\huge.log").openReader(StandardCharsets.UTF_8)) { + reader.lines().filter(line -> line.contains("ERROR")).forEach(System.out::println); +} +``` + +The stream decodes the content block by block as it arrives (48 KiB per block): memory stays +bounded whatever the file size. Like a [`RemoteProcess`](commands.html), **it must be closed** — +it holds the client's connection until it reaches its end or is closed, and closing it early stops +the remote read. The file is opened before `openStream()` returns, so a missing file fails there; +a failure midway is reported by `read()`, never as a silently short read. + +## Timeouts + +The blocking terminals (`readBytes`, `readText`, `digest`) run under a **wall-clock deadline**: +the client's timeout, or `timeout(Duration)` on the request — raise it for large reads. +`openStream()`/`openReader()` use the **inactivity** semantics of the other streaming terminals: +the timeout bounds the silence between two blocks, not the whole read. See +[Timeouts and Errors](timeouts-and-errors.html). + +## Locked files, errors and requirements + +* The file is opened with a share mode that tolerates other writers, so **a log being written by + a running service can be read**. A file held with an exclusive lock (`pagefile.sys`, live + registry hives) fails with a *sharing violation*. +* Every failure is a `WinRMClientException` naming its cause: file not found, the path is a + directory, access denied, sharing violation, PowerShell not available, or PowerShell in + Constrained Language Mode. +* The host needs **PowerShell 2.0 or later in `FullLanguage` mode** (Windows Server 2008 R2 and + later ship it). When AppLocker or WDAC puts PowerShell in Constrained Language Mode, the .NET + calls the reader relies on are blocked: remote file access is then not available. +* Non-ASCII paths and content are safe: the path travels base64-encoded (UTF-8) inside the script + and the content comes back base64-encoded, so neither depends on the remote console code page. + +## Performance + +The mechanism is designed for **configuration files, logs and small data files — not bulk data**. +Measured over HTTP with NTLM encryption, `openStream()` reads about **1.4–1.5 MB/s**: a 20 MiB +file in 14–15 seconds, on Windows Server 2008 R2 (PowerShell 2.0), 2019 and 2022 (PowerShell 5.1) +alike. A small file costs about a second, mostly the PowerShell startup. For gigabytes, use SMB. + +The limit is on the host, not in the network or the client: the WinRM service reads a command's +output pipe itself, **at most 32 KiB per read and about 60 reads per second** per output stream — +about 2 MB/s of output whatever the command. The reader is shaped for it: each block of the file +travels as one 65,533-byte base64 line that fills exactly two reads (a larger 77,825-byte line +needing three reads was 20–25% slower). Neither the client's `MaxEnvelopeSize` nor the host's +`MaxEnvelopeSizekb` changes the limit — raising them only makes each response larger and +proportionally slower to come. + +The same limit applies to any command output: a command that writes its output in small pieces — +line by line, like `Write-Output` or `type` — gets only what accumulated in the 4 KiB pipe at each +read, about 0.3 MB/s. Parallel commands, each on its own `WinRMClient`, add up: each command gets +its own output pipeline. + +## See also + +* [File Transfers](file-transfers.html) — the other direction: copying local files to the host +* [Remote Commands](commands.html) — the command shell the reads ride diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md index 1ef7114..ee31e31 100644 --- a/src/site/markdown/index.md +++ b/src/site/markdown/index.md @@ -126,6 +126,7 @@ remain available and unchanged, with their checked exceptions. * [WQL Queries](wql.html) — query WMI and read the result * [Remote Commands](commands.html) — run commands and copy files to the host * [File Transfers](file-transfers.html) — how files are copied through the WinRM channel +* [Remote Files](files.html) — read remote files: whole, byte ranges, tails, streams, digests * [Command-Line Client](cli.html) — the standalone jar's manual page * [Authentication](authentication.html) — NTLM and Kerberos * [TLS / HTTPS](tls.html) — certificate validation and trust stores diff --git a/src/site/site.xml b/src/site/site.xml index 2591475..38f54fc 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -57,6 +57,7 @@ + diff --git a/src/test/java/org/metricshub/winrm/RemoteFileTest.java b/src/test/java/org/metricshub/winrm/RemoteFileTest.java new file mode 100644 index 0000000..230842f --- /dev/null +++ b/src/test/java/org/metricshub/winrm/RemoteFileTest.java @@ -0,0 +1,258 @@ +package org.metricshub.winrm; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * WinRM Java Client + * ჻჻჻჻჻჻ + * Copyright (C) 2023 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.metricshub.winrm.light.FakeWsmanResponses.commandResponse; +import static org.metricshub.winrm.light.FakeWsmanResponses.done; +import static org.metricshub.winrm.light.FakeWsmanResponses.envelope; +import static org.metricshub.winrm.light.FakeWsmanResponses.receiveResponse; +import static org.metricshub.winrm.light.FakeWsmanResponses.resourceCreated; +import static org.metricshub.winrm.light.FakeWsmanResponses.signalResponse; +import static org.metricshub.winrm.light.FakeWsmanResponses.stream; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.metricshub.winrm.exceptions.WinRMClientException; +import org.metricshub.winrm.exceptions.WinRMTimeoutException; +import org.metricshub.winrm.light.FakeWsmanServer; + +/** + * Client-side tests of {@link RemoteFile} against {@link FakeWsmanServer}: the scripted + * base64-line output is decoded incrementally, exit codes become the documented exceptions, and + * the request settings reach the script. The scripts themselves run in + * {@link RemoteFilesScriptTest}. + */ +class RemoteFileTest { + + private static final String DOMAIN = "FAKE"; + private static final String USER = "user"; + private static final String PASSWORD = "s3cret-Passw0rd"; + private static final String COMMAND_ID = "CMD-1"; + private static final String PATH = "C:\\Windows\\Temp\\collect.bin"; + + private FakeWsmanServer server; + + @BeforeEach + void startServer() throws Exception { + server = new FakeWsmanServer(DOMAIN, USER, PASSWORD); + } + + @AfterEach + void stopServer() { + server.close(); + } + + private WinRMClient client() { + return WinRMClient + .builder("127.0.0.1") + .port(server.port()) + .credentials(DOMAIN + "\\" + USER, PASSWORD.toCharArray()) + .timeout(Duration.ofSeconds(10)) + .build(); + } + + private static String b64(final byte[] bytes) { + return Base64.getEncoder().encodeToString(bytes); + } + + private static String stdout(final String text) { + return stream("stdout", COMMAND_ID, text.getBytes(StandardCharsets.US_ASCII)); + } + + /** Script a whole read: shell creation, command, the given Receive chunks (the last one completes), Signal. */ + private void enqueueRead(final int exitCode, final String stderr, final String... chunks) { + server.enqueue(200, envelope(resourceCreated("SHELL-1"))).enqueue(200, envelope(commandResponse(COMMAND_ID))); + for (int i = 0; i < chunks.length; i++) { + final boolean last = i == chunks.length - 1; + final String streams = stdout(chunks[i]) + + (last && stderr != null ? stream("stderr", COMMAND_ID, stderr.getBytes(StandardCharsets.UTF_8)) : ""); + server.enqueue(200, envelope(receiveResponse(streams, last ? done(COMMAND_ID, exitCode) : null))); + } + server.enqueue(200, envelope(signalResponse())); + } + + /** The PowerShell script the client sent, decoded from its -EncodedCommand. */ + private String sentScript() { + final String command = server + .decryptedRequests() + .stream() + .filter(r -> r.contains("-EncodedCommand")) + .findFirst() + .orElseThrow(() -> new AssertionError("no PowerShell command sent")); + final Matcher matcher = Pattern.compile("-EncodedCommand ([A-Za-z0-9+/=]+)").matcher(command); + assertTrue(matcher.find(), command); + return new String(Base64.getDecoder().decode(matcher.group(1)), StandardCharsets.UTF_16LE); + } + + @Test + void readBytesDecodesLinesSplitAcrossReceiveChunks() { + final byte[] content = new byte[300]; + for (int i = 0; i < content.length; i++) { + content[i] = (byte) i; + } + final String line1 = b64(java.util.Arrays.copyOfRange(content, 0, 150)); + final String line2 = b64(java.util.Arrays.copyOfRange(content, 150, 300)); + // The first line is cut in the middle by the protocol chunking. + enqueueRead(0, null, line1.substring(0, 17), line1.substring(17) + "\r\n" + line2 + "\r\n"); + + try (WinRMClient client = client()) { + assertArrayEquals(content, client.file(PATH).readBytes()); + } + final String script = sentScript(); + // The path travels base64-encoded: no quoting of caller input, whatever it contains. + assertTrue(script.contains(b64(PATH.getBytes(StandardCharsets.UTF_8))), script); + // Whole file from the start, capped one byte past the default maxBytes. + assertTrue(script.contains("$n=[long]0;$l=[long]" + (RemoteFile.DEFAULT_MAX_BYTES + 1)), script); + } + + @Test + void rangeSettingsReachTheScript() { + enqueueRead(0, null, b64("tail".getBytes(StandardCharsets.US_ASCII)) + "\r\n"); + try (WinRMClient client = client()) { + assertEquals("tail", client.file(PATH).offset(-8192).length(1024).readText(StandardCharsets.US_ASCII)); + } + assertTrue(sentScript().contains("$n=[long]-8192;$l=[long]1024"), sentScript()); + } + + @Test + void openStreamHasNoCapAndReadsToTheEnd() throws Exception { + enqueueRead(0, null, b64(new byte[] { 1, 2, 3 }) + "\r\n", b64(new byte[] { 4, 5 }) + "\r\n"); + try (WinRMClient client = client(); InputStream in = client.file(PATH).openStream()) { + assertArrayEquals(new byte[] { 1, 2, 3, 4, 5 }, in.readAllBytes()); + } + assertTrue(sentScript().contains("$n=[long]0;$l=[long]-1"), sentScript()); + } + + @Test + void readTextKeepsTheBomAndOpenReaderDecodesACharacterSplitAcrossLines() throws Exception { + final byte[] bom = { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }; + enqueueRead(0, null, b64(bom) + "\r\n" + b64("é".getBytes(StandardCharsets.UTF_8)) + "\r\n"); + try (WinRMClient client = client()) { + assertEquals("\uFEFFé", client.file(PATH).readText(StandardCharsets.UTF_8)); + } + + // "é" is C3 A9: its two bytes arrive on two separate lines (two transfer blocks). + server.close(); + server = new FakeWsmanServer(DOMAIN, USER, PASSWORD); + enqueueRead(0, null, b64(new byte[] { 'a', (byte) 0xC3 }) + "\r\n", b64(new byte[] { (byte) 0xA9, 'b' }) + "\r\n"); + try (WinRMClient client = client(); BufferedReader reader = client.file(PATH).openReader(StandardCharsets.UTF_8)) { + assertEquals("aéb", reader.readLine()); + } + } + + @Test + void theSizeCapIsSentToTheHostAndEnforced() { + enqueueRead(0, null, b64(new byte[] { 1, 2, 3, 4, 5 }) + "\r\n"); + try (WinRMClient client = client()) { + final WinRMClientException e = assertThrows( + WinRMClientException.class, + () -> client.file(PATH).maxBytes(4).readBytes() + ); + assertTrue(e.getMessage().contains("4-byte cap"), e.getMessage()); + } + assertTrue(sentScript().contains("$l=[long]5"), sentScript()); + } + + @Test + void aMissingFileFailsWhenTheStreamIsOpened() { + enqueueRead(RemoteFiles.EXIT_NOT_FOUND, "Could not find file", ""); + try (WinRMClient client = client()) { + final RemoteFile file = client.file(PATH); + final WinRMClientException e = assertThrows(WinRMClientException.class, file::openStream); + assertTrue(e.getMessage().contains("not found") && e.getMessage().contains(PATH), e.getMessage()); + } + } + + @Test + void exitCodesBecomeExplicitFailures() { + assertTrue(RemoteFiles.failure(3, PATH, "h", "").getMessage().contains("sharing violation")); + assertTrue(RemoteFiles.failure(4, PATH, "h", "").getMessage().contains("Access denied")); + assertTrue(RemoteFiles.failure(5, PATH, "h", "").getMessage().contains("Constrained Language Mode")); + assertTrue(RemoteFiles.failure(6, PATH, "h", "").getMessage().contains("is a directory")); + assertTrue(RemoteFiles.failure(9009, PATH, "h", "").getMessage().contains("PowerShell is not available")); + final String other = RemoteFiles.failure(1, PATH, "h", " I/O device error \n").getMessage(); + assertTrue(other.contains("exit code 1") && other.endsWith(": I/O device error"), other); + } + + @Test + void aFailureMidwayIsNotASilentlyShortRead() throws Exception { + enqueueRead(1, "The device is not ready", b64(new byte[] { 1, 2, 3 }) + "\r\n"); + try (WinRMClient client = client(); InputStream in = client.file(PATH).openStream()) { + assertEquals(1, in.read()); + final WinRMClientException e = assertThrows(WinRMClientException.class, in::readAllBytes); + assertTrue(e.getMessage().contains("device is not ready"), e.getMessage()); + } + } + + @Test + void unexpectedOutputIsReported() { + enqueueRead(0, null, "WARNING: not base64!\r\n"); + try (WinRMClient client = client()) { + final WinRMClientException e = assertThrows(WinRMClientException.class, () -> client.file(PATH).readBytes()); + assertTrue(e.getMessage().contains("Unexpected output"), e.getMessage()); + } + } + + @Test + void digestIsLowercaseHex() { + enqueueRead(0, null, b64(new byte[] { (byte) 0xAB, 0x01, (byte) 0xFF }) + "\r\n"); + try (WinRMClient client = client()) { + assertEquals("ab01ff", client.file(PATH).digest("SHA256")); + } + assertTrue(sentScript().contains("HashAlgorithm]::Create('SHA256')"), sentScript()); + } + + @Test + void invalidSettingsAreRejectedLocally() { + try (WinRMClient client = client()) { + final RemoteFile file = client.file(PATH); + assertThrows(IllegalArgumentException.class, () -> file.length(-1)); + assertThrows(IllegalArgumentException.class, () -> file.maxBytes(-1)); + assertThrows(IllegalArgumentException.class, () -> file.maxBytes(Integer.MAX_VALUE)); + assertThrows(IllegalArgumentException.class, () -> file.digest("sha256; rm")); + assertThrows(IllegalArgumentException.class, () -> client.file(" ")); + } + assertEquals(0, server.decryptedRequests().size()); + } + + @Test + void theBlockingTerminalsHonorTheWallClockTimeout() { + server.enqueue(200, envelope(resourceCreated("SHELL-1"))).enqueue(200, envelope(commandResponse(COMMAND_ID))); + server.enqueueDelayed(200, envelope(receiveResponse("", done(COMMAND_ID, 0))), 3_000); + server.enqueue(200, envelope(signalResponse())); + try (WinRMClient client = client()) { + final RemoteFile file = client.file(PATH).timeout(Duration.ofMillis(500)); + assertThrows(WinRMTimeoutException.class, file::readBytes); + } + } +} diff --git a/src/test/java/org/metricshub/winrm/RemoteFilesScriptTest.java b/src/test/java/org/metricshub/winrm/RemoteFilesScriptTest.java new file mode 100644 index 0000000..7a60d28 --- /dev/null +++ b/src/test/java/org/metricshub/winrm/RemoteFilesScriptTest.java @@ -0,0 +1,196 @@ +package org.metricshub.winrm; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * WinRM Java Client + * ჻჻჻჻჻჻ + * Copyright (C) 2023 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Base64; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +/** + * Runs the PowerShell scripts of {@link RemoteFiles} in the local {@code powershell.exe}, + * exactly as the remote shell would ({@code -EncodedCommand}), and decodes their output the way + * the client does: the actual script semantics (seek, negative offsets, share mode, exit codes) + * are only testable against a real PowerShell. Windows only. + */ +@EnabledOnOs(OS.WINDOWS) +class RemoteFilesScriptTest { + + /** Every byte value, then enough random bytes to span several transfer blocks. */ + private static byte[] content; + + @TempDir + static Path directory; + + private static Path file; + + @BeforeAll + static void createFile() throws Exception { + content = new byte[256 + RemoteFiles.BLOCK_SIZE * 2 + 1234]; + for (int i = 0; i < 256; i++) { + content[i] = (byte) i; + } + final byte[] random = new byte[content.length - 256]; + new Random(42).nextBytes(random); + System.arraycopy(random, 0, content, 256, random.length); + file = directory.resolve("données-漢字.bin"); + Files.write(file, content); + } + + /** The outcome of a script: the decoded bytes and the exit code. */ + private static final class Outcome { + + final byte[] bytes; + final int exitCode; + + Outcome(final byte[] bytes, final int exitCode) { + this.bytes = bytes; + this.exitCode = exitCode; + } + } + + private static Outcome run(final String script) throws Exception { + final String encoded = Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_16LE)); + final Process process = new ProcessBuilder( + "powershell.exe", + "-NoProfile", + "-NonInteractive", + "-EncodedCommand", + encoded + ) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start(); + process.getOutputStream().close(); + final String stdout = new String(process.getInputStream().readAllBytes(), StandardCharsets.US_ASCII); + if (!process.waitFor(60, TimeUnit.SECONDS)) { + process.destroyForcibly(); + throw new AssertionError("powershell.exe did not complete"); + } + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + for (final String line : stdout.split("\r?\n")) { + if (!line.isBlank()) { + bytes.write(Base64.getDecoder().decode(line.strip())); + } + } + return new Outcome(bytes.toByteArray(), process.exitValue()); + } + + private static byte[] read(final long offset, final long length) throws Exception { + final Outcome outcome = run(RemoteFiles.readScript(file.toString(), offset, length)); + assertEquals(0, outcome.exitCode); + return outcome.bytes; + } + + private static byte[] slice(final int from, final int to) { + return Arrays.copyOfRange(content, from, to); + } + + @Test + void wholeFileIsByteExactAcrossBlocksAndWithANonAsciiName() throws Exception { + assertArrayEquals(content, read(0, -1)); + } + + @Test + void rangesSeekAndClampAtTheEnd() throws Exception { + assertArrayEquals(slice(100, 200), read(100, 100)); + // A range spanning a block boundary. + assertArrayEquals( + slice(RemoteFiles.BLOCK_SIZE - 10, RemoteFiles.BLOCK_SIZE + 10), + read(RemoteFiles.BLOCK_SIZE - 10, 20) + ); + // Length beyond the end: what exists. + assertArrayEquals(slice(content.length - 5, content.length), read(content.length - 5, 1000)); + // Past the end, and a zero length: empty, not an error. + assertEquals(0, read(content.length + 10, -1).length); + assertEquals(0, read(10, 0).length); + } + + @Test + void negativeOffsetsCountFromTheEnd() throws Exception { + assertArrayEquals(slice(content.length - 8192, content.length), read(-8192, -1)); + assertArrayEquals(slice(content.length - 8192, content.length - 8192 + 1024), read(-8192, 1024)); + // Equal to, and larger than, the size: the whole file. + assertArrayEquals(content, read(-content.length, -1)); + assertArrayEquals(content, read(-content.length - 1000L, -1)); + } + + @Test + void emptyFile() throws Exception { + final Path empty = Files.write(directory.resolve("empty.txt"), new byte[0]); + final Outcome outcome = run(RemoteFiles.readScript(empty.toString(), 0, -1)); + assertEquals(0, outcome.exitCode); + assertEquals(0, outcome.bytes.length); + } + + @Test + void aFileOpenForWritingByAnotherProcessCanBeRead() throws Exception { + final Path log = directory.resolve("service.log"); + try (FileOutputStream writer = new FileOutputStream(log.toFile())) { + writer.write("line 1\r\n".getBytes(StandardCharsets.US_ASCII)); + writer.flush(); + assertArrayEquals( + "line 1\r\n".getBytes(StandardCharsets.US_ASCII), + run(RemoteFiles.readScript(log.toString(), 0, -1)).bytes + ); + } + } + + @Test + void failuresMapToTheDocumentedExitCodes() throws Exception { + assertEquals( + RemoteFiles.EXIT_NOT_FOUND, + run(RemoteFiles.readScript(directory.resolve("missing.txt").toString(), 0, -1)).exitCode + ); + assertEquals( + RemoteFiles.EXIT_NOT_FOUND, + run(RemoteFiles.readScript(directory.resolve("no\\such\\dir.txt").toString(), 0, -1)).exitCode + ); + assertEquals(RemoteFiles.EXIT_IS_DIRECTORY, run(RemoteFiles.readScript(directory.toString(), 0, -1)).exitCode); + + final Path locked = Files.write(directory.resolve("locked.bin"), new byte[] { 1, 2, 3 }); + try (RandomAccessFile raf = new RandomAccessFile(locked.toFile(), "rw"); FileLock lock = raf.getChannel().lock()) { + assertEquals(RemoteFiles.EXIT_SHARING_VIOLATION, run(RemoteFiles.readScript(locked.toString(), 0, -1)).exitCode); + } + } + + @Test + void digestMatchesTheLocalDigest() throws Exception { + final Outcome outcome = run(RemoteFiles.digestScript(file.toString(), "SHA256")); + assertEquals(0, outcome.exitCode); + assertArrayEquals(MessageDigest.getInstance("SHA-256").digest(content), outcome.bytes); + } +} diff --git a/src/test/java/org/metricshub/winrm/WinRMLiveTest.java b/src/test/java/org/metricshub/winrm/WinRMLiveTest.java index 9ca595a..71273b0 100644 --- a/src/test/java/org/metricshub/winrm/WinRMLiveTest.java +++ b/src/test/java/org/metricshub/winrm/WinRMLiveTest.java @@ -1,5 +1,6 @@ package org.metricshub.winrm; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -7,6 +8,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.MessageDigest; +import java.time.Duration; import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -112,4 +115,81 @@ void commandSucceeds() throws Exception { assertEquals(0, result.getStatusCode(), () -> "stderr: " + result.getStderr()); assertTrue(result.getStdout().length() > 0, "command must produce stdout"); } + + private static WinRMClient client() { + final WinRMClient.Builder builder = WinRMClient.builder(host).credentials(username, password) + .timeout(Duration.ofSeconds(60)); + if (protocol == WinRMHttpProtocolEnum.HTTPS) { + builder.https(); + } + if (port != null) { + builder.port(port); + } + return builder.build(); + } + + @Test + void remoteFileReadsAreByteExact() throws Exception { + // Every byte value, over several transfer blocks. Created on the host by one command: an + // uploadFile of that size needs more command legs than old hosts' operation quota allows. + final byte[] content = new byte[200_000]; + for (int i = 0; i < content.length; i++) { + content[i] = (byte) (i % 251); + } + final String remote = "C:\\Windows\\Temp\\winrm-java-live-read-\u00e9.bin"; + final StringBuilder sha256 = new StringBuilder(); + for (final byte b : MessageDigest.getInstance("SHA-256").digest(content)) { + sha256.append(String.format("%02x", b)); + } + + try (WinRMClient client = client()) { + client + .powerShell( + "$b=New-Object byte[] " + content.length + ";for($i=0;$i -lt $b.Length;$i++){$b[$i]=$i % 251}" + + ";[IO.File]::WriteAllBytes('" + remote + "',$b)" + ) + .execute(); + try { + final long start = System.nanoTime(); + assertArrayEquals(content, client.file(remote).readBytes()); + System.out.printf("Read %d bytes in %d ms%n", content.length, (System.nanoTime() - start) / 1_000_000L); + assertArrayEquals( + java.util.Arrays.copyOfRange(content, content.length - 8192, content.length - 8192 + 1024), + client.file(remote).offset(-8192).length(1024).readBytes() + ); + assertEquals(sha256.toString(), client.file(remote).digest("SHA256")); + } finally { + client.powerShell("Remove-Item -LiteralPath '" + remote + "' -ErrorAction SilentlyContinue").execute(); + } + } + } + + @Test + void remoteFileRangeIsServedBySeekingAndAFileOpenForWritingCanBeRead() throws Exception { + final String big = "C:\\Windows\\Temp\\winrm-java-live-big.bin"; + final String log = "C:\\Windows\\Temp\\winrm-java-live-open.log"; + try (WinRMClient client = client(); WinRMClient writer = client()) { + // 1 GiB of zeros: reading it whole through base64 would take many minutes. + client + .powerShell( + "$f=[IO.File]::Create('" + big + "');$f.SetLength(1073741824);$f.Close();" + + "[IO.File]::WriteAllText('" + log + "','first line')" + ) + .execute(); + final long start = System.nanoTime(); + assertEquals(4096, client.file(big).offset(-4096).readBytes().length); + final long seconds = (System.nanoTime() - start) / 1_000_000_000L; + assertTrue(seconds < 20, "the tail of a 1 GiB file must be read by a seek, took " + seconds + " s"); + + // Another session holds the log open for writing (sharing reads only, like most services). + try ( + RemoteProcess holder = writer + .powerShell("$f=[IO.File]::Open('" + log + "','Open','Write','Read');Start-Sleep 60") + .start()) { + Thread.sleep(3_000); + assertEquals("first line", client.file(log).readText(StandardCharsets.US_ASCII)); + } + client.command("del /q \"" + big + "\" \"" + log + "\"").execute(); + } + } } From c64286047e950c9127701ca48d1527e64fe3c166 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 24 Sep 2026 14:18:18 +0200 Subject: [PATCH 2/2] Address Codex review: exact-length reads, long paths, BOM wording, README - readBytes(): a read that fills exactly the requested length now drains the stream to its end, so the script's exit code is still checked and a late failure is not reported as a successful range read. - A path too long for the reader script to fit the remote command line (about 1,450 ASCII characters) is refused with a WinRMClientException before anything is sent, instead of silently falling back to uploading the script: a read never writes on the host. Documented in files.md. - readText()/openReader() Javadoc: nothing is stripped, a byte order mark is handled as the given charset's decoder handles it (UTF-16 consumes it). - README: revert the quick-start addition (AGENTS.md keeps README terse; the feature is documented in files.md). Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ---- .../org/metricshub/winrm/CommandRequest.java | 2 +- .../java/org/metricshub/winrm/RemoteFile.java | 19 +++++++++++---- .../org/metricshub/winrm/RemoteFiles.java | 11 +++++++++ src/site/markdown/files.md | 4 ++++ .../org/metricshub/winrm/RemoteFileTest.java | 24 +++++++++++++++++++ 6 files changed, 55 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e23dc5d..a40ab5d 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,6 @@ through the unchecked `WinRMClientException` hierarchy (`WinRMAuthenticationExce `WqlSyntaxException`). ```java -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Duration; import org.metricshub.winrm.*; @@ -99,9 +98,6 @@ try (WinRMClient client = WinRMClient.builder("server01.acme.com") // Copy a file to the host (through the WinRM channel itself — no SMB) client.uploadFile(Path.of("collect.ps1"), "C:\\Windows\\Temp\\collect.ps1"); - - // Read a remote file — whole, a byte range, or the tail of a log - String tail = client.file("D:\\logs\\app.log").offset(-8192).readText(StandardCharsets.UTF_8); } ``` diff --git a/src/main/java/org/metricshub/winrm/CommandRequest.java b/src/main/java/org/metricshub/winrm/CommandRequest.java index a26ff5c..d38a70a 100644 --- a/src/main/java/org/metricshub/winrm/CommandRequest.java +++ b/src/main/java/org/metricshub/winrm/CommandRequest.java @@ -154,7 +154,7 @@ private interface StdinSource { * shell's command-line limit (room for the {@code CMD.EXE /C} wrapper included, so * the decision does not depend on {@link #upload(Path...)}) */ - private static String encodePowerShell(final String script) { + static String encodePowerShell(final String script) { final String encoded = POWERSHELL_PREFIX + Base64.getEncoder().encodeToString(script.getBytes(StandardCharsets.UTF_16LE)); return encoded.length() > MAX_COMMAND_LINE_LENGTH - CMD_WRAPPER_LENGTH ? null : encoded; diff --git a/src/main/java/org/metricshub/winrm/RemoteFile.java b/src/main/java/org/metricshub/winrm/RemoteFile.java index bfa79fb..79200cb 100644 --- a/src/main/java/org/metricshub/winrm/RemoteFile.java +++ b/src/main/java/org/metricshub/winrm/RemoteFile.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.nio.charset.Charset; import java.time.Duration; import java.util.concurrent.Callable; @@ -57,7 +58,10 @@ *

* The file is opened with a share mode that tolerates other writers, so a log being written by a * running service can be read; a file held with an exclusive lock (e.g. {@code pagefile.sys}) - * fails with a sharing violation. Ranges are byte ranges, and reads are not snapshots: a + * fails with a sharing violation. A read writes nothing on the host: its script travels on the + * command line, which limits the path to about 1,450 characters (fewer with non-Latin + * characters) — a longer path fails before anything is sent. Ranges are byte ranges, and + * reads are not snapshots: a * file that grows or shrinks between two reads is read as it is at each read. *

* A request is not thread-safe; configure it and call its terminals from one thread. @@ -193,6 +197,10 @@ public byte[] readBytes() { ) ); } + // A read that filled exactly the requested length stops short of the end of the + // stream: drain it (the host sends nothing past the length) so the script's exit code + // is still checked and a late failure is not reported as a successful read. + in.transferTo(OutputStream.nullOutputStream()); return content; } }); @@ -200,8 +208,10 @@ public byte[] readBytes() { /** * Read the content (the whole file, or the configured range) into memory and decode it with the - * given charset — no guessing, no default. A byte order mark is not stripped (it - * decodes to {@code U+FEFF}). A range boundary may split a multibyte character, which then + * given charset — no guessing, no default. This method strips nothing: a byte order mark is + * handled exactly as the given charset's decoder handles it — {@code UTF-8} keeps it (as + * {@code U+FEFF}), while {@code UTF-16} consumes it to detect the byte order. A range boundary + * may split a multibyte character, which then * decodes to {@code U+FFFD} at the edges, like any malformed input. * * @param charset the charset the file is encoded with @@ -237,7 +247,8 @@ public InputStream openStream() { /** * Open the content (the whole file, or the configured range) as a character stream decoded * with the given charset: {@link #openStream()} behind an {@link InputStreamReader}, with the - * same lifecycle — close it. A byte order mark is not stripped. + * same lifecycle — close it. Like {@link #readText(Charset)}, it strips nothing: a byte + * order mark is handled as the given charset's decoder handles it. * * @param charset the charset the file is encoded with * @return the reader, to use with try-with-resources diff --git a/src/main/java/org/metricshub/winrm/RemoteFiles.java b/src/main/java/org/metricshub/winrm/RemoteFiles.java index e75ec93..fa35a6b 100644 --- a/src/main/java/org/metricshub/winrm/RemoteFiles.java +++ b/src/main/java/org/metricshub/winrm/RemoteFiles.java @@ -174,6 +174,17 @@ private static String openFile(final String path) { * @return the decoded stream; it must be closed */ static InputStream open(final WinRMClient client, final String path, final String script, final Duration timeout) { + if (CommandRequest.encodePowerShell(script) == null) { + // powerShell(...) would transparently fall back to uploading the script as a file: a read + // must stay read-only on the host (no files written, no certutil), so refuse instead. + throw new WinRMClientException( + String.format( + "Remote path too long to read on %s (%d characters): the reader script must fit the remote command line", + client.hostname(), + path.length() + ) + ); + } final RemoteProcess process = client.powerShell(script).timeout(timeout).start(); try { return new DecodingStream(process, path, client.hostname()); diff --git a/src/site/markdown/files.md b/src/site/markdown/files.md index 75d50b9..13dcc5e 100644 --- a/src/site/markdown/files.md +++ b/src/site/markdown/files.md @@ -96,6 +96,10 @@ the timeout bounds the silence between two blocks, not the whole read. See calls the reader relies on are blocked: remote file access is then not available. * Non-ASCII paths and content are safe: the path travels base64-encoded (UTF-8) inside the script and the content comes back base64-encoded, so neither depends on the remote console code page. +* A read never writes anything on the host: the reader script travels on the command line, which + limits the path to about **1,450 characters** (about 720 with accented letters, 480 with CJK + characters — well above the classic 260-character `MAX_PATH`). A longer path fails with a + `WinRMClientException` before anything is sent. ## Performance diff --git a/src/test/java/org/metricshub/winrm/RemoteFileTest.java b/src/test/java/org/metricshub/winrm/RemoteFileTest.java index 230842f..5e835d4 100644 --- a/src/test/java/org/metricshub/winrm/RemoteFileTest.java +++ b/src/test/java/org/metricshub/winrm/RemoteFileTest.java @@ -204,6 +204,30 @@ void exitCodesBecomeExplicitFailures() { assertTrue(other.contains("exit code 1") && other.endsWith(": I/O device error"), other); } + @Test + void anExactLengthReadStillChecksTheExitCode() { + // The host sends exactly the requested bytes, then fails (e.g. while closing the file): + // readBytes() must drain to the end of the output instead of returning a full buffer. + enqueueRead(1, "The device is not ready", b64(new byte[] { 1, 2, 3 }) + "\r\n"); + try (WinRMClient client = client()) { + final RemoteFile file = client.file(PATH).length(3); + final WinRMClientException e = assertThrows(WinRMClientException.class, file::readBytes); + assertTrue(e.getMessage().contains("device is not ready"), e.getMessage()); + } + } + + @Test + void aPathTooLongForTheCommandLineIsRefusedBeforeAnythingIsSent() { + try (WinRMClient client = client()) { + final RemoteFile file = client.file("C:\\" + "a".repeat(1500)); + final WinRMClientException e = assertThrows(WinRMClientException.class, file::readBytes); + assertTrue(e.getMessage().contains("too long"), e.getMessage()); + // Just under the limit still fits: the script is not uploaded as a file. + assertTrue(CommandRequest.encodePowerShell(RemoteFiles.readScript("C:\\" + "a".repeat(1450), -8192, 1)) != null); + } + assertEquals(0, server.decryptedRequests().size()); + } + @Test void aFailureMidwayIsNotASilentlyShortRead() throws Exception { enqueueRead(1, "The device is not ready", b64(new byte[] { 1, 2, 3 }) + "\r\n");