diff --git a/changes.md b/changes.md index b8192b962a..b11842b2dd 100644 --- a/changes.md +++ b/changes.md @@ -189,6 +189,11 @@ Changes log - The HTTP client connects directly to the hosts listed in the "nonProxyHosts" parameter or the "http.nonProxyHosts" system property, without proxy. As with the JDK, the loopback addresses are excluded by default. Issue #401. + - The HTTP client supports SOCKS5 proxies, with optional username/password + authentication: "socksProxyHost", "socksProxyPort", "socksProxyUsername" and + "socksProxyPassword" parameters, defaulting to the "socksProxyHost", "socksProxyPort", + "java.net.socks.username" and "java.net.socks.password" system properties. An HTTP + proxy takes precedence. Issue #1573. - Added an optional origin matcher to CORS (`CorsService`, `CorsFilter`, `setAllowedOriginMatcher(Predicate)`) to accept origins that are not listed, for example any sub-domain. The response always contains the exact request origin. Issue #1611. - Added the `SameSite` attribute to cookie settings (`CookieSetting.get/setSameSite()`, `SameSite` enum), read and written in `Set-Cookie`. diff --git a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java index 2ee520217f..3cf4cf1de4 100644 --- a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java +++ b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientCall.java @@ -331,7 +331,7 @@ void handle(org.restlet.Request request, org.restlet.Response response) { if (isTunneled()) { moveTunnelProxyAuthorization(request); } else if (!isForwardProxied()) { - // Direct connection: the proxy credentials must not reach the origin server. + // Direct or SOCKS connection: the proxy credentials must not reach the origin server. requestHeaders.removeAll(HeaderConstants.HEADER_PROXY_AUTHORIZATION, true); } diff --git a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java index 7170fd436d..8bc26ad5ab 100644 --- a/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java +++ b/org.restlet/src/main/java/org/restlet/engine/netty/NettyClientHelper.java @@ -35,6 +35,8 @@ import io.netty.handler.codec.http2.Http2StreamFrameToHttpObjectCodec; import io.netty.handler.proxy.HttpProxyHandler; import io.netty.handler.proxy.ProxyConnectException; +import io.netty.handler.proxy.ProxyHandler; +import io.netty.handler.proxy.Socks5ProxyHandler; import io.netty.handler.ssl.ApplicationProtocolConfig; import io.netty.handler.ssl.ApplicationProtocolNames; import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler; @@ -171,7 +173,7 @@ * nonProxyHosts * String * System property "http.nonProxyHosts" or "localhost|127.*|[::1]|0.0.0.0|[::0]" - * The hosts to connect to directly, without proxy, separated by "|". A host name can start or + * The hosts to connect to directly, without HTTP or SOCKS proxy, separated by "|". A host name can start or * end with a "*" wildcard, e.g. "localhost|*.example.com|192.168.*", and is matched ignoring * case. As with the JDK, the loopback addresses are excluded by default, and an empty value * excludes no host. @@ -197,6 +199,32 @@ * The size in bytes of the buffer used to write request entities * * + * socksProxyHost + * String + * System property "socksProxyHost" + * The host name of the SOCKS5 proxy, used for the requests without HTTP proxy. The SOCKS + * proxy resolves the host names of the origin servers. + * + * + * socksProxyPort + * int + * System property "socksProxyPort" or "1080" + * The port of the SOCKS5 proxy. + * + * + * socksProxyUsername + * String + * System property "java.net.socks.username" + * The user name for the username/password authentication with the SOCKS5 proxy, if + * required. + * + * + * socksProxyPassword + * String + * System property "java.net.socks.password" + * The password for the username/password authentication with the SOCKS5 proxy. + * + * * userAgentField * String * null @@ -485,7 +513,8 @@ private static void resetResponse(Response response, Request next) { Channel acquire(NettyClientCall call) throws IOException { if (call.isConfidential() && getTransportMode() == TransportMode.HTTP3 - && getProxy(true, call.getTargetHost()) == null) { + && getProxy(true, call.getTargetHost()) == null + && getSocksProxy(call.getTargetHost()) == null) { return acquireHttp3(call); } @@ -680,11 +709,9 @@ private Channel connect(final OriginPool pool, final String proxyAuthorization) boolean acquired = true; try { final SslContext ssl = pool.secure ? getSslContext() : null; - final InetSocketAddress proxyRef = getProxy(pool.secure, pool.host); - final InetSocketAddress proxy = - (proxyRef == null) - ? null - : new InetSocketAddress(proxyRef.getHostString(), proxyRef.getPort()); + final InetSocketAddress proxy = resolve(getProxy(pool.secure, pool.host)); + final InetSocketAddress socksProxy = + (proxy == null) ? resolve(getSocksProxy(pool.host)) : null; Bootstrap bootstrap = new Bootstrap() @@ -700,7 +727,12 @@ private Channel connect(final OriginPool pool, final String proxyAuthorization) @Override protected void initChannel(SocketChannel ch) { initConnection( - ch, pool, ssl, proxy, proxyAuthorization); + ch, + pool, + ssl, + proxy, + proxyAuthorization, + socksProxy); } }); @@ -710,8 +742,8 @@ protected void initChannel(SocketChannel ch) { } SocketAddress remote; - if (proxy != null && pool.secure) { - // Tunnel with CONNECT, the proxy resolves the host name. + if ((proxy != null && pool.secure) || (socksProxy != null)) { + // Tunnel with CONNECT or SOCKS, the proxy resolves the host name. bootstrap.resolver(NoopAddressResolverGroup.INSTANCE); remote = InetSocketAddress.createUnresolved(pool.host, pool.port); } else if (proxy != null) { @@ -806,7 +838,8 @@ private void initConnection( OriginPool pool, SslContext ssl, InetSocketAddress proxy, - String proxyAuthorization) { + String proxyAuthorization, + InetSocketAddress socksProxy) { final Promise ready = ch.eventLoop().newPromise(); ch.attr(READY).set(ready); ch.attr(POOL).set(pool); @@ -820,16 +853,22 @@ private void initConnection( } p.addLast("writability", Writability.HANDLER); - if ((proxy == null) || !pool.secure) { + ProxyHandler proxyHandler; + if (socksProxy != null) { + proxyHandler = + new Socks5ProxyHandler( + socksProxy, getSocksProxyUsername(), getSocksProxyPassword()); + } else if ((proxy != null) && pool.secure) { + HttpHeaders connectHeaders = new DefaultHttpHeaders(); + if (proxyAuthorization != null) { + connectHeaders.set(HttpHeaderNames.PROXY_AUTHORIZATION, proxyAuthorization); + } + proxyHandler = new HttpProxyHandler(proxy, connectHeaders); + } else { initProtocols(ch, pool, ssl, ready, proxy != null); return; } - HttpHeaders connectHeaders = new DefaultHttpHeaders(); - if (proxyAuthorization != null) { - connectHeaders.set(HttpHeaderNames.PROXY_AUTHORIZATION, proxyAuthorization); - } - HttpProxyHandler proxyHandler = new HttpProxyHandler(proxy, connectHeaders); proxyHandler.setConnectTimeoutMillis(getConnectTimeout()); p.addLast("proxy", proxyHandler); p.addLast( @@ -1166,6 +1205,73 @@ InetSocketAddress getProxy(boolean secure, String targetHost) { host, secure ? getHttpsProxyPort() : getProxyPort()); } + /** + * Returns the SOCKS5 proxy to connect to an origin server through, or null to connect directly. + * An HTTP proxy, if any for the request, takes precedence. + * + * @param targetHost The host name of the origin server. + * @return The unresolved address of the SOCKS5 proxy, or null. + */ + InetSocketAddress getSocksProxy(String targetHost) { + String host = getSocksProxyHost(); + return ((host == null) || host.isEmpty() || isNonProxyHost(targetHost)) + ? null + : InetSocketAddress.createUnresolved(host, getSocksProxyPort()); + } + + /** + * Returns the host name of the SOCKS5 proxy, if specified: the "socksProxyHost" parameter, or + * the "socksProxyHost" system property. + * + * @return The host name of the SOCKS5 proxy, if specified. + */ + public String getSocksProxyHost() { + return getHelpedParameters() + .getFirstValue("socksProxyHost", System.getProperty("socksProxyHost")); + } + + /** + * Returns the port of the SOCKS5 proxy: the "socksProxyPort" parameter, or the "socksProxyPort" + * system property, 1080 otherwise. + * + * @return The port of the SOCKS5 proxy. + */ + public int getSocksProxyPort() { + return Integer.parseInt( + getHelpedParameters() + .getFirstValue( + "socksProxyPort", System.getProperty("socksProxyPort", "1080"))); + } + + /** + * Returns the user name for the authentication with the SOCKS5 proxy, if specified: the + * "socksProxyUsername" parameter, or the "java.net.socks.username" system property. + * + * @return The user name, or null. + */ + public String getSocksProxyUsername() { + return getHelpedParameters() + .getFirstValue("socksProxyUsername", System.getProperty("java.net.socks.username")); + } + + /** + * Returns the password for the authentication with the SOCKS5 proxy, if specified: the + * "socksProxyPassword" parameter, or the "java.net.socks.password" system property. + * + * @return The password, or null. + */ + public String getSocksProxyPassword() { + return getHelpedParameters() + .getFirstValue("socksProxyPassword", System.getProperty("java.net.socks.password")); + } + + /** Resolves the host name of a proxy address, if any. */ + private static InetSocketAddress resolve(InetSocketAddress address) { + return (address == null) + ? null + : new InetSocketAddress(address.getHostString(), address.getPort()); + } + /** * Returns the hosts to connect to directly, without proxy, separated by "|": the * "nonProxyHosts" parameter, or the "http.nonProxyHosts" system property, the loopback diff --git a/org.restlet/src/test/java/org/restlet/engine/connector/SocksProxyTestCase.java b/org.restlet/src/test/java/org/restlet/engine/connector/SocksProxyTestCase.java new file mode 100644 index 0000000000..daa85691b3 --- /dev/null +++ b/org.restlet/src/test/java/org/restlet/engine/connector/SocksProxyTestCase.java @@ -0,0 +1,441 @@ +/** + * Copyright 2005-2026 Qlik + *

+ * The content of this file is subject to the terms of the Apache 2.0 open + * source license available at https://www.opensource.org/licenses/apache-2.0 + *

+ * Restlet is a registered trademark of QlikTech International AB. + */ +package org.restlet.engine.connector; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.restlet.Client; +import org.restlet.Context; +import org.restlet.Request; +import org.restlet.Response; +import org.restlet.Restlet; +import org.restlet.Server; +import org.restlet.data.ChallengeResponse; +import org.restlet.data.ChallengeScheme; +import org.restlet.data.MediaType; +import org.restlet.data.Method; +import org.restlet.data.Protocol; +import org.restlet.data.Status; +import org.restlet.engine.Engine; +import org.restlet.engine.header.HeaderConstants; + +/** Tests the SOCKS5 proxy support of the client connector, for HTTP and HTTPS requests. */ +class SocksProxyTestCase { + + @TempDir Path tempDir; + + private TestSocksProxy proxy; + + private Server server; + + private Client client; + + @BeforeEach + void setUp() throws Exception { + Engine.clearThreadLocalVariables(); + Engine.register(false); + Engine.getInstance().getRegisteredServers().add(new HttpServerHelper(null)); + Engine.getInstance().getRegisteredServers().add(new HttpsServerHelper(null)); + Engine.getInstance().getRegisteredClients().add(new HttpClientHelper(null)); + proxy = new TestSocksProxy(); + } + + @AfterEach + void tearDown() throws Exception { + if (client != null) { + client.stop(); + } + if (server != null) { + server.stop(); + } + proxy.close(); + Engine.clearThreadLocalVariables(); + } + + private static final Restlet ECHO = + new Restlet() { + @Override + public void handle(Request request, Response response) { + String proxyAuthorization = + request.getHeaders() + .getFirstValue( + HeaderConstants.HEADER_PROXY_AUTHORIZATION, true); + response.setEntity( + request.getResourceRef().getScheme() + + " " + + request.getResourceRef().getPath() + + ((proxyAuthorization == null) + ? "" + : " " + proxyAuthorization), + MediaType.TEXT_PLAIN); + } + }; + + @Test + void connectsHttpRequestsThroughSocksProxy() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, ECHO); + server.start(); + + Context context = socksContext(); + client = new Client(context, Protocol.HTTP); + client.start(); + + for (int i = 0; i < 2; i++) { + Response response = + client.handle( + new Request( + Method.GET, + "http://localhost:" + server.getActualPort() + "/a?b=c")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertEquals("http /a", response.getEntityAsText()); + } + + // The connection is reused, and the proxy resolves the host name. + assertEquals(List.of("localhost:" + server.getActualPort()), proxy.targets); + } + + @Test + void connectsHttpsRequestsThroughSocksProxyWithCredentials() throws Exception { + proxy.requiredCredentials = "scott:tiger"; + Path keystore = keystore(); + startHttpsServer(keystore); + + Context context = socksContext(); + context.getParameters().add("socksProxyUsername", "scott"); + context.getParameters().add("socksProxyPassword", "tiger"); + addTrustStore(context, keystore); + client = new Client(context, Protocol.HTTPS); + client.start(); + + Response response = + client.handle( + new Request( + Method.GET, "https://localhost:" + server.getActualPort() + "/s")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertEquals("https /s", response.getEntityAsText()); + assertEquals(List.of("localhost:" + server.getActualPort()), proxy.targets); + } + + @Test + void doesNotSendProxyCredentialsToOriginServer() throws Exception { + Path keystore = keystore(); + startHttpsServer(keystore); + Context context = socksContext(); + addTrustStore(context, keystore); + client = new Client(context, List.of(Protocol.HTTP, Protocol.HTTPS)); + client.start(); + Server httpServer = new Server(new Context(), Protocol.HTTP, 0, ECHO); + httpServer.start(); + + try { + for (String uri : + List.of( + "http://localhost:" + httpServer.getActualPort() + "/p", + "https://localhost:" + server.getActualPort() + "/p")) { + Request request = new Request(Method.GET, uri); + ChallengeResponse credentials = + new ChallengeResponse(new ChallengeScheme("HTTP_BEARER", "Bearer")); + credentials.setRawValue("token"); + request.setProxyChallengeResponse(credentials); + Response response = client.handle(request); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + // No "Proxy-Authorization" header echoed. + assertEquals( + uri.substring(0, uri.indexOf(':')) + " /p", response.getEntityAsText()); + } + } finally { + httpServer.stop(); + } + assertEquals(2, proxy.targets.size()); + } + + @Test + void ignoresEmptySocksProxyHost() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, ECHO); + server.start(); + + Context context = socksContext(); + context.getParameters().set("socksProxyHost", ""); + client = new Client(context, Protocol.HTTP); + client.start(); + + Response response = + client.handle( + new Request( + Method.GET, "http://localhost:" + server.getActualPort() + "/e")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertTrue(proxy.targets.isEmpty(), proxy.targets.toString()); + } + + @Test + void reportsRefusedSocksCredentials() throws Exception { + proxy.requiredCredentials = "scott:tiger"; + server = new Server(new Context(), Protocol.HTTP, 0, ECHO); + server.start(); + + Context context = socksContext(); + context.getParameters().add("socksProxyUsername", "scott"); + context.getParameters().add("socksProxyPassword", "wrong"); + client = new Client(context, Protocol.HTTP); + client.start(); + + Response response = + client.handle( + new Request(Method.GET, "http://localhost:" + server.getActualPort())); + assertTrue(response.getStatus().isConnectorError(), response.getStatus().toString()); + assertTrue(proxy.targets.isEmpty(), proxy.targets.toString()); + } + + @Test + void bypassesSocksProxyForNonProxyHosts() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, ECHO); + server.start(); + + Context context = socksContext(); + context.getParameters().set("nonProxyHosts", "localhost"); + client = new Client(context, Protocol.HTTP); + client.start(); + + Response response = + client.handle( + new Request( + Method.GET, "http://localhost:" + server.getActualPort() + "/d")); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + assertTrue(proxy.targets.isEmpty(), proxy.targets.toString()); + } + + @Test + void usesSocksProxySystemProperties() throws Exception { + server = new Server(new Context(), Protocol.HTTP, 0, ECHO); + server.start(); + + String previousHost = System.setProperty("socksProxyHost", "localhost"); + String previousPort = System.setProperty("socksProxyPort", Integer.toString(proxy.port())); + String previousNonProxyHosts = System.setProperty("http.nonProxyHosts", ""); + try { + client = new Client(new Context(), Protocol.HTTP); + client.start(); + Response response = + client.handle( + new Request(Method.GET, "http://localhost:" + server.getActualPort())); + assertEquals(Status.SUCCESS_OK, response.getStatus()); + } finally { + restore("socksProxyHost", previousHost); + restore("socksProxyPort", previousPort); + restore("http.nonProxyHosts", previousNonProxyHosts); + } + + assertEquals(List.of("localhost:" + server.getActualPort()), proxy.targets); + } + + private static void restore(String name, String value) { + if (value == null) { + System.clearProperty(name); + } else { + System.setProperty(name, value); + } + } + + private Context socksContext() { + Context context = new Context(); + context.getParameters().add("socksProxyHost", "localhost"); + context.getParameters().add("socksProxyPort", Integer.toString(proxy.port())); + context.getParameters().add("nonProxyHosts", ""); + return context; + } + + private Path keystore() throws IOException { + Path keystore = tempDir.resolve("dummy.p12"); + try (InputStream in = SslBaseConnectorsTestCase.class.getResourceAsStream("dummy.p12")) { + Files.copy(in, keystore, REPLACE_EXISTING); + } + return keystore; + } + + private void startHttpsServer(Path keystore) throws Exception { + Context serverContext = new Context(); + serverContext.getParameters().add("keystorePath", keystore.toString()); + serverContext.getParameters().add("keystorePassword", "testtest"); + serverContext.getParameters().add("keyStoreType", "PKCS12"); + serverContext.getParameters().add("keyPassword", "testtest"); + server = new Server(serverContext, Protocol.HTTPS, 0, ECHO); + server.start(); + } + + private static void addTrustStore(Context context, Path keystore) { + context.getParameters().add("truststorePath", keystore.toString()); + context.getParameters().add("truststorePassword", "testtest"); + context.getParameters().add("trustStoreType", "PKCS12"); + } + + /** + * Minimal SOCKS5 proxy (RFC 1928), with optional username/password authentication (RFC 1929), + * supporting the CONNECT command only. + */ + private static final class TestSocksProxy implements AutoCloseable { + + /** The "host:port" targets of the CONNECT commands accepted. */ + final List targets = new CopyOnWriteArrayList<>(); + + /** The "user:password" credentials required, if any. */ + volatile String requiredCredentials; + + private final ServerSocket serverSocket; + + private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor(); + + TestSocksProxy() throws IOException { + serverSocket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); + executor.execute(this::acceptLoop); + } + + int port() { + return serverSocket.getLocalPort(); + } + + private void acceptLoop() { + try { + while (!serverSocket.isClosed()) { + Socket socket = serverSocket.accept(); + executor.execute(() -> handle(socket)); + } + } catch (IOException _) { + // closed + } + } + + private void handle(Socket client) { + try (client) { + DataInputStream in = new DataInputStream(client.getInputStream()); + OutputStream out = client.getOutputStream(); + + // Method selection. + in.readUnsignedByte(); + byte[] methods = in.readNBytes(in.readUnsignedByte()); + int method = (requiredCredentials == null) ? 0x00 : 0x02; + if (!contains(methods, method)) { + out.write(new byte[] {0x05, (byte) 0xFF}); + return; + } + out.write(new byte[] {0x05, (byte) method}); + out.flush(); + + if (method == 0x02 && !authenticate(in, out)) { + return; + } + + // CONNECT request. + in.readUnsignedByte(); + int command = in.readUnsignedByte(); + in.readUnsignedByte(); + String host = + switch (in.readUnsignedByte()) { + case 0x01 -> + InetAddress.getByAddress(in.readNBytes(4)).getHostAddress(); + case 0x03 -> + new String( + in.readNBytes(in.readUnsignedByte()), + StandardCharsets.US_ASCII); + default -> InetAddress.getByAddress(in.readNBytes(16)).getHostAddress(); + }; + int port = in.readUnsignedShort(); + if (command != 0x01) { + out.write(new byte[] {0x05, 0x07, 0x00, 0x01, 0, 0, 0, 0, 0, 0}); + return; + } + + try (Socket origin = new Socket(host, port)) { + targets.add(host + ":" + port); + out.write(new byte[] {0x05, 0x00, 0x00, 0x01, 0, 0, 0, 0, 0, 0}); + out.flush(); + relay(client, in, origin); + } + } catch (IOException _) { + // connection closed + } + } + + private boolean authenticate(DataInputStream in, OutputStream out) throws IOException { + in.readUnsignedByte(); + String user = new String(in.readNBytes(in.readUnsignedByte()), StandardCharsets.UTF_8); + String password = + new String(in.readNBytes(in.readUnsignedByte()), StandardCharsets.UTF_8); + boolean accepted = requiredCredentials.equals(user + ":" + password); + out.write(new byte[] {0x01, (byte) (accepted ? 0x00 : 0x01)}); + out.flush(); + return accepted; + } + + private static boolean contains(byte[] values, int value) { + for (byte b : values) { + if ((b & 0xFF) == value) { + return true; + } + } + return false; + } + + /** Relays bytes both ways until one side closes. */ + private static void relay(Socket client, InputStream clientIn, Socket origin) + throws IOException { + Thread upstream = + Thread.ofVirtual() + .start( + () -> { + try { + clientIn.transferTo(origin.getOutputStream()); + } catch (IOException _) { + // closed + } finally { + try { + origin.shutdownOutput(); + } catch (IOException _) { + // ignore + } + } + }); + try { + origin.getInputStream().transferTo(client.getOutputStream()); + } finally { + client.shutdownOutput(); + try { + upstream.join(5000); + } catch (InterruptedException _) { + Thread.currentThread().interrupt(); + } + } + } + + @Override + public void close() throws IOException { + serverSocket.close(); + executor.shutdownNow(); + } + } +}