HTTP Basic: decode credentials as UTF-8, advertise charset, read token68 challenges

changes.md

@@ -77,6 +77,14 @@ Changes log
           explicitly. Without a valid configuration, `CorsFilter` adds no CORS headers. Issue #1611.
         - CORS: `CorsFilter` now applies configuration changes made after the first request (the
           internal `CorsResponseHelper` is rebuilt by the setters). Issue #1611.
+        - HTTP Basic: the credentials sent by a client are now decoded as UTF-8, falling back to
+          ISO-8859-1 when the bytes are not valid UTF-8. A `ChallengeAuthenticator` using
+          `ChallengeScheme.HTTP_BASIC` now advertises `charset="UTF-8"` in its challenge (RFC 7617),
+          and a `ChallengeResponse` built from a challenge request copies that `charset`
+          parameter, so the client encodes its credentials in UTF-8. The charsets can be changed per
+          authenticator with `ChallengeAuthenticator.setCredentialsCharset()` (UTF-8 by default,
+          advertised in the challenge only when it is UTF-8) and `setCredentialsFallbackCharset()`
+          (ISO-8859-1 by default, null to reject invalid UTF-8). Issue #1455.
         - STDIO connector construction fails fast without an explicit or uniquely registered
           supporting helper class. Applications with multiple STDIO protocol helpers must
           select one explicitly.
@@ -160,6 +168,10 @@ Changes log
         - `CookieAuthenticator` no longer lets the next restlet overwrite the redirection sent after
           a login. Issue #554.
         - CorsService and CorsFilter now add `Vary: Origin` to every response when the CORS answer depends on the request origin (origin allow-list or credentials allowed), including requests without an `Origin` header, so shared caches no longer serve the response of one origin to another. A static `*` without credentials still sends no `Vary`, as recommended by the Fetch standard. Issue #1611.
+        - Fixed the parsing of a `WWW-Authenticate` challenge made of a token68 ending with `=`
+          (such as a `Negotiate` token): the whole header was dropped. Issue #191.
+        - The `realm` and `charset` parameters of an HTTP Basic challenge are now recognized whatever
+          their case (`Realm=`, `Charset=`), as required by RFC 9110. Issue #1455.
     - Misc
         - Fixed JDK HTTP client accessibility in core and WebSocket tests in Eclipse and VS Code,
           without adding a production module dependency.

org.restlet/src/main/java/org/restlet/engine/header/ChallengeRequestReader.java

@@ -54,6 +54,13 @@ public HeaderWriter<Parameter> append(Parameter value) {
                         }
                     };
 
+            // A challenge may carry a single token68 instead of parameters
+            // (RFC 9110, section 11.2), such as a base64 Negotiate token.
+            String token68 = readToken68();
+            if (token68 != null) {
+                w.append(token68);
+            }
+
             boolean stop = false;
             while (peek() != -1 && !stop) {
                 boolean sepSkipped = skipValueSeparator();
@@ -82,4 +89,48 @@ public HeaderWriter<Parameter> append(Parameter value) {
 
         return result;
     }
+
+    /**
+     * Reads a token68 ending with at least one "=" padding character, if the next characters are
+     * one. A token68 without padding is read as a regular parameter, as before.
+     *
+     * @return The token68, or null (nothing consumed) if the next characters are not one.
+     */
+    private String readToken68() {
+        mark();
+        StringBuilder token = new StringBuilder();
+        int next = peek();
+        while (next != -1 && isToken68Char(next)) {
+            token.append((char) read());
+            next = peek();
+        }
+
+        int padding = 0;
+        while (next == '=') {
+            token.append((char) read());
+            padding++;
+            next = peek();
+        }
+
+        if (token.length() > padding
+                && padding > 0
+                && (next == -1 || next == ',' || isSpace(next))) {
+            return token.toString();
+        }
+
+        reset();
+        return null;
+    }
+
+    private static boolean isToken68Char(int c) {
+        return (c >= 'A' && c <= 'Z')
+                || (c >= 'a' && c <= 'z')
+                || (c >= '0' && c <= '9')
+                || c == '-'
+                || c == '.'
+                || c == '_'
+                || c == '~'
+                || c == '+'
+                || c == '/';
+    }
 }

org.restlet/src/main/java/org/restlet/engine/security/AuthenticatorUtils.java

@@ -37,6 +37,8 @@
  */
 public class AuthenticatorUtils {
 
+    private static final String CHARSET_PARAMETER = "charset";
+
     /**
      * Indicates if any of the objects is null.
      *
@@ -362,6 +364,14 @@ public static void update(
             realm = challengeRequest.getRealm();
             nonce = challengeRequest.getServerNonce();
             challengeResponse.setOpaque(challengeRequest.getOpaque());
+
+            // Credentials charset advertised by the server (RFC 7617, section 2.1)
+            String charset = challengeRequest.getParameters().getFirstValue(CHARSET_PARAMETER, true);
+            if (ChallengeScheme.HTTP_BASIC.equals(challengeResponse.getScheme())
+                    && charset != null
+                    && challengeResponse.getParameters().getFirstValue(CHARSET_PARAMETER, true) == null) {
+                challengeResponse.getParameters().add(CHARSET_PARAMETER, charset);
+            }
         }
 
         challengeResponse.setRealm(realm);

org.restlet/src/main/java/org/restlet/engine/security/HttpBasicHelper.java

@@ -11,6 +11,11 @@
 import java.io.CharArrayWriter;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.logging.Level;
 import org.restlet.Context;
@@ -50,7 +55,7 @@ public void formatRequest(
             Response response,
             Series<Header> httpHeaders) {
         String realm = challenge.getRealm();
-        String charset = challenge.getParameters().getFirstValue(CHARSET);
+        String charset = challenge.getParameters().getFirstValue(CHARSET, true);
 
         if (realm != null) {
             cw.appendQuotedChallengeParameter(REALM, realm);
@@ -122,7 +127,7 @@ public void parseRequest(
 
         while (param != null) {
             try {
-                if (REALM.equals(param.getName())) {
+                if (REALM.equalsIgnoreCase(param.getName())) {
                     challenge.setRealm(param.getValue());
                 } else {
                     challenge.getParameters().add(param);
@@ -152,31 +157,66 @@ public void parseResponse(
         }
 
         try {
-            String charset = getCharset(challenge);
-
-            byte[] credentialsEncoded = Base64.getDecoder().decode(challenge.getRawValue());
-
-            String credentials = new String(credentialsEncoded, charset);
-            int separator = credentials.indexOf(':');
-
-            if (separator == -1) {
+            if (!parseCredentials(challenge, StandardCharsets.UTF_8, StandardCharsets.ISO_8859_1)) {
                 // Log the blocking
                 getLogger()
                         .log(
                                 Level.INFO,
                                 "Invalid credentials given by client with IP: {0}",
                                 request != null ? request.getClientInfo().getAddress() : "?");
-            } else {
-                challenge.setIdentifier(credentials.substring(0, separator));
-                challenge.setSecret(credentials.substring(separator + 1));
             }
-        } catch (UnsupportedEncodingException e) {
-            getLogger().log(Level.INFO, "Unsupported HTTP Basic encoding error", e);
         } catch (IllegalArgumentException e) {
             getLogger().log(Level.INFO, "Unable to decode the HTTP Basic credential", e);
         }
     }
 
+    /**
+     * Decodes the Base64 credentials of a challenge response and sets its identifier and secret.
+     * The server advertises {@code charset="UTF-8"} (RFC 7617 section 2.1) but the header does not
+     * tell which charset the client used, so the bytes are decoded with the primary charset and, if
+     * they are not valid in it, with the fallback charset.
+     *
+     * @param challenge The challenge response, with its raw value set.
+     * @param primary The charset tried first.
+     * @param fallback The charset used if the bytes are not valid in the primary one, or null for
+     *     none.
+     * @return True if the identifier and secret were set, false if the decoded credentials have no
+     *     ":" separator.
+     * @throws IllegalArgumentException If the raw value is not valid Base64 or the bytes are not
+     *     valid in the primary charset without a fallback.
+     */
+    public static boolean parseCredentials(
+            ChallengeResponse challenge, Charset primary, Charset fallback) {
+        byte[] bytes = Base64.getDecoder().decode(challenge.getRawValue());
+        String credentials;
+
+        try {
+            credentials =
+                    primary.newDecoder()
+                            .onMalformedInput(CodingErrorAction.REPORT)
+                            .onUnmappableCharacter(CodingErrorAction.REPORT)
+                            .decode(ByteBuffer.wrap(bytes))
+                            .toString();
+        } catch (CharacterCodingException e) {
+            if (fallback == null) {
+                throw new IllegalArgumentException(
+                        "The credentials are not valid in " + primary.name(), e);
+            }
+
+            credentials = new String(bytes, fallback);
+        }
+
+        int separator = credentials.indexOf(':');
+
+        if (separator == -1) {
+            return false;
+        }
+
+        challenge.setIdentifier(credentials.substring(0, separator));
+        challenge.setSecret(credentials.substring(separator + 1));
+        return true;
+    }
+
     /**
      * Returns the charset from the given ChallengeResponse. If the charset is not specified, or if
      * it is not UTF-8, then ISO-8859-1 is returned as per RFC 7617.
@@ -185,7 +225,7 @@ public void parseResponse(
      * @return The charset.
      */
     private String getCharset(final ChallengeResponse challenge) {
-        String charset = challenge.getParameters().getFirstValue(CHARSET);
+        String charset = challenge.getParameters().getFirstValue(CHARSET, true);
 
         if (charset == null) {
             charset = ISO_8859_1;

org.restlet/src/main/java/org/restlet/security/ChallengeAuthenticator.java

@@ -8,6 +8,9 @@
  */
 package org.restlet.security;
 
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
 import java.util.logging.Level;
 import org.restlet.Context;
 import org.restlet.Request;
@@ -17,6 +20,7 @@
 import org.restlet.data.ChallengeScheme;
 import org.restlet.data.ClientInfo;
 import org.restlet.data.Status;
+import org.restlet.engine.security.HttpBasicHelper;
 
 /**
  * Authenticator based on a challenge scheme. This is typically used to support the HTTP BASIC and
@@ -29,6 +33,18 @@
  */
 public class ChallengeAuthenticator extends Authenticator {
 
+    /**
+     * The charset of the HTTP Basic credentials, advertised in the challenge when it is UTF-8 (RFC
+     * 7617).
+     */
+    private volatile Charset credentialsCharset = StandardCharsets.UTF_8;
+
+    /**
+     * The charset used when the HTTP Basic credentials are not valid in the credentials charset, or
+     * null for none.
+     */
+    private volatile Charset credentialsFallbackCharset = StandardCharsets.ISO_8859_1;
+
     /** The authentication realm. */
     private volatile String realm;
 
@@ -120,6 +136,8 @@ protected boolean authenticate(Request request, Response response) {
         boolean loggable = request.isLoggable() && getLogger().isLoggable(Level.FINE);
 
         if (getVerifier() != null) {
+            decodeBasicCredentials(request);
+
             switch (getVerifier().verify(request, response)) {
                 case Verifier.RESULT_VALID:
                     // Valid credentials provided
@@ -222,7 +240,15 @@ public void challenge(Response response, boolean stale) {
      * @return A new challenge request.
      */
     protected ChallengeRequest createChallengeRequest(boolean stale) {
-        return new ChallengeRequest(getScheme(), getRealm());
+        ChallengeRequest result = new ChallengeRequest(getScheme(), getRealm());
+
+        if (ChallengeScheme.HTTP_BASIC.equals(getScheme())
+                && StandardCharsets.UTF_8.equals(getCredentialsCharset())) {
+            // RFC 7617 section 2.1: the only charset that can be advertised is UTF-8
+            result.getParameters().add("charset", "UTF-8");
+        }
+
+        return result;
     }
 
     /**
@@ -247,6 +273,51 @@ public void forbid(Response response) {
         response.setStatus(Status.CLIENT_ERROR_FORBIDDEN);
     }
 
+    /**
+     * Decodes again the HTTP Basic credentials with the charsets of this authenticator, when they
+     * differ from the default decoding (UTF-8, then ISO-8859-1).
+     *
+     * @param request The request holding the challenge response.
+     */
+    private void decodeBasicCredentials(Request request) {
+        ChallengeResponse cr = request.getChallengeResponse();
+
+        if (cr != null
+                && ChallengeScheme.HTTP_BASIC.equals(cr.getScheme())
+                && cr.getRawValue() != null
+                && (!StandardCharsets.UTF_8.equals(credentialsCharset)
+                        || !StandardCharsets.ISO_8859_1.equals(credentialsFallbackCharset))) {
+            try {
+                HttpBasicHelper.parseCredentials(
+                        cr, credentialsCharset, credentialsFallbackCharset);
+            } catch (IllegalArgumentException e) {
+                getLogger().log(Level.INFO, "Unable to decode the HTTP Basic credential", e);
+                cr.setIdentifier(null);
+                cr.setSecret((char[]) null);
+            }
+        }
+    }
+
+    /**
+     * Returns the charset of the HTTP Basic credentials. Defaults to UTF-8.
+     *
+     * @return The credentials charset.
+     * @see #setCredentialsCharset(Charset)
+     */
+    public Charset getCredentialsCharset() {
+        return credentialsCharset;
+    }
+
+    /**
+     * Returns the charset used when the HTTP Basic credentials are not valid in the credentials
+     * charset. Defaults to ISO-8859-1.
+     *
+     * @return The fallback charset, or null if there is none.
+     */
+    public Charset getCredentialsFallbackCharset() {
+        return credentialsFallbackCharset;
+    }
+
     /**
      * Returns the authentication realm.
      *
@@ -285,6 +356,28 @@ public boolean isRechallenging() {
         return this.rechallenging;
     }
 
+    /**
+     * Sets the charset of the HTTP Basic credentials (RFC 7617 section 2.1). The credentials are
+     * read with it, and the challenge advertises {@code charset="UTF-8"} when it is UTF-8, the only
+     * value allowed by the RFC. Defaults to UTF-8.
+     *
+     * @param credentialsCharset The credentials charset.
+     */
+    public void setCredentialsCharset(Charset credentialsCharset) {
+        this.credentialsCharset = Objects.requireNonNull(credentialsCharset);
+    }
+
+    /**
+     * Sets the charset used when the HTTP Basic credentials are not valid in the credentials
+     * charset, as clients that predate RFC 7617 send ISO-8859-1. Defaults to ISO-8859-1. Use null
+     * to reject such credentials.
+     *
+     * @param credentialsFallbackCharset The fallback charset, or null for none.
+     */
+    public void setCredentialsFallbackCharset(Charset credentialsFallbackCharset) {
+        this.credentialsFallbackCharset = credentialsFallbackCharset;
+    }
+
     /**
      * Sets the authentication realm.
      *