From 8de41b50248aec125d3b206c8210839e154e66c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Sun, 15 Nov 2026 23:56:00 +0100 Subject: [PATCH] initial implementation of refresh tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- README.rst | 92 ++++++- src/flexmeasures_client/client.py | 140 ++++++++-- src/flexmeasures_client/response_handling.py | 3 +- tests/client/test_init.py | 10 +- tests/client/test_refresh_tokens.py | 272 +++++++++++++++++++ 5 files changed, 487 insertions(+), 30 deletions(-) create mode 100644 tests/client/test_refresh_tokens.py diff --git a/README.rst b/README.rst index 895f1121..af665330 100644 --- a/README.rst +++ b/README.rst @@ -67,15 +67,39 @@ To get started with the FlexMeasures Client, first an account needs to be regist To create a local instance of FlexMeasures, follow the `FlexMeasures documentation `_. Registering to a hosted FlexMeasures instance instead can be done through `Seita BV `_. -In these examples we show how to set up the client to connect to either ``http://localhost:5000`` or ``https://ems.seita.energy``. To connect to a different host, adapt the host in the initialization of the client. +For an interactive session, create a client without credentials. On the first +request that needs authentication, it prompts for an email address and a +password (without echoing the password): - .. code-block:: python +.. code-block:: python from flexmeasures_client import FlexMeasuresClient - async def main(): - client = FlexMeasuresClient(host="localhost:5000", ssl=False, email="email@email.com", password="pw") - client = FlexMeasuresClient(host="ems.seita.energy", ssl=True, email="email@email.com", password="pw") + client = FlexMeasuresClient(host="ems.seita.energy", ssl=True) + # Use the client, then close its HTTP session: + # await client.get_assets() + # await client.close() + +You can instead pass ``email`` and ``password`` to the constructor, including +for unattended programs: + +.. code-block:: python + + client = FlexMeasuresClient( + host="ems.seita.energy", ssl=True, email=email, password=password, + ) + +``host="localhost:5000", ssl=False`` connects to a local development server. + +With an existing refresh token, you can authenticate without a password: + +.. code-block:: python + + client = FlexMeasuresClient( + host="ems.seita.energy", ssl=True, refresh_token=saved_token, + ) + +See `Refresh tokens and secret storage`_ for how to save and reuse tokens. Retrieving available info @@ -270,6 +294,64 @@ advanced options (training window, regressors, forecast frequency, etc.) see :doc:`forecasting`. +.. _refresh-token-storage: + +Refresh tokens and secret storage +================================= + +By default the client requests a refresh token at login. It exchanges that +for a new access and refresh token when an API request receives a 401 or +``get_access_token()`` is called. If the refresh token is rejected, it logs in +again: supplied credentials are used if available, otherwise it prompts for +missing credentials. A process without an interactive terminal must provide +credentials or a valid refresh token. Older servers continue to use password +login. + +To reuse a token across runs, an application can use `keyring +`_ and a configured system keyring. +Install ``keyring`` separately. This example identifies the secret by server +and account, so each account has its own entry: + +.. code-block:: python + + from functools import partial + import keyring + from flexmeasures_client import FlexMeasuresClient + + service = "flexmeasures:ems.seita.energy" + account = "email@example.com" # keyring entry name, not a password + saved_token = keyring.get_password(service, account) + save_token = partial(keyring.set_password, service, account) + + client = FlexMeasuresClient( + host="ems.seita.energy", ssl=True, email=account, + refresh_token=saved_token, + refresh_token_callback=save_token, + ) + # Use the client, then await client.close(). + +If the entry is missing or rejected, the client prompts for the account password +and the callback saves the resulting refresh token. The email can also be +omitted when using a token; both credentials are then requested if login is +needed. The callback receives the new token; +``partial`` binds the service and account names, and other storage backends can +bind whatever additional context they need. The callback runs synchronously after +every issuance or rotation, and must replace the previous token. If it raises, +the error propagates and the new token remains on ``client.refresh_token`` for +recovery. Select a secure keyring backend appropriate to your environment. + +Set ``use_refresh_tokens=False`` when constructing the client to disable refresh +tokens and use password authentication. The client itself keeps credentials in +memory and does not write them to disk. Protect refresh tokens, use TLS, and do +not share a refresh token between client instances or processes: concurrent +exchanges can invalidate the token family. The client coordinates concurrent +requests within one instance only. + +A refresh request whose response is lost may already have rotated the token. +The client does not automatically retry a timed-out refresh; the application may +need to log in again. + + Development ============== diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index 9252c477..f8d344e8 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -1,11 +1,13 @@ from __future__ import annotations import asyncio +import getpass import json import logging import os import re import socket +import sys import time import warnings from dataclasses import InitVar, dataclass, field @@ -161,14 +163,22 @@ def _describe_failed_job(job_id: str, job: dict) -> str: class FlexMeasuresClient: """Main class for connecting to the FlexMeasures API.""" - password: str - email: str + password: str | None = field(default=None, repr=False) + email: str | None = None host: str = "localhost:5000" port: int | None = None ssl: bool = False api_version: str = API_VERSION path: str = f"/api/{api_version}/" - access_token: str | None = None + access_token: str | None = field(default=None, repr=False) + use_refresh_tokens: bool = field(default=True, kw_only=True) + refresh_token: str | None = field(default=None, repr=False, kw_only=True) + refresh_token_callback: Callable[[str], None] | None = field( + default=None, repr=False, kw_only=True + ) + _auth_lock: asyncio.Lock = field( + default_factory=asyncio.Lock, init=False, repr=False + ) # These govern the HTTP request/retry loop used throughout the client: auth, # version discovery, assets and sensors, data I/O, trigger requests, result @@ -242,7 +252,7 @@ def __post_init__( if self.session is None: self.session = ClientSession() - if not re.match(r".+\@.+\..+", self.email): + if self.email is not None and not re.match(r".+\@.+\..+", self.email): raise EmailValidationError( f"{self.email} is not an email address format string" ) @@ -270,7 +280,7 @@ def __post_init__( f"https: // should not be included in {self.host}." f"To use https: // set ssl=True and host={host_without_scheme}" ) - if len(self.password) < 1: + if self.password == "": raise EmptyPasswordError("password cannot be empty") self.determine_port() @@ -342,6 +352,7 @@ async def request( minimum_server_version: str | None = None, minimum_server_version_msg: str | None = None, pass_through_statuses: frozenset[int] = frozenset(), + retry_on_timeout: bool = True, ) -> tuple[dict | list, int]: """Send a request to FlexMeasures. @@ -408,6 +419,7 @@ def report_rate_limit(message: str) -> None: reauth_once=reauth_once, pass_through_statuses=pass_through_statuses, rate_limit_callback=report_rate_limit, + auth_token_used=headers.get("Authorization"), ) if ( response.status < 300 @@ -430,6 +442,10 @@ def report_rate_limit(message: str) -> None: self.logger.info(message) break except asyncio.TimeoutError: + if not retry_on_timeout: + raise ConnectionError( + "Request timed out; its outcome is unknown." + ) sleep_interval = self.request_retry_interval * (2**polling_step) message = f"Client request timeout occurred while connecting to the API. Polling step: {polling_step}. Retrying in {sleep_interval} seconds..." # noqa: E501 self.logger.debug(message) @@ -477,12 +493,13 @@ async def request_once( reauth_once: bool = True, pass_through_statuses: frozenset[int] = frozenset(), rate_limit_callback: Callable[[str], None] | None = None, + auth_token_used: str | None = None, ) -> tuple[ClientResponse, int, bool, URL]: url_msg = f"url: {url}" - json_msg = f"payload: {json_payload}" + json_msg = "payload: [redacted]" params_msg = f"params: {params}" method_msg = f"method: {method}" - headers_msg = f"headers: {headers}" + headers_msg = "headers: [redacted]" self.logger.debug("===== Request =====") self.logger.debug(url_msg) self.logger.debug(json_msg) @@ -506,9 +523,9 @@ async def request_once( ssl=self.ssl, allow_redirects=False, ) - payload = await response.json() + await response.json() status_msg = f"status: {response.status}" - response_payload_msg = f"payload: {payload}" + response_payload_msg = "payload: [redacted]" headers_msg = f"headers: {response.headers}" self.logger.debug("===== Response =====") @@ -538,6 +555,7 @@ async def request_once( method=method, pass_through_statuses=pass_through_statuses, rate_limit_callback=rate_limit_callback, + auth_token_used=auth_token_used, ) return response, polling_step, reauth_once, url @@ -564,7 +582,7 @@ async def get_headers(self, include_auth: bool) -> dict: if include_auth: access_token = self.access_token if access_token is None: - access_token = await self.get_access_token() + access_token = await self._authenticate(None) headers |= {"Authorization": access_token} return headers @@ -577,25 +595,101 @@ def build_url(self, uri: str, path: str = path) -> URL: ) return url - async def get_access_token(self) -> str: - """Get access token, store it on the FlexMeasuresClient, and return it.""" - response, _status = await self.request( - uri="requestAuthToken", - path="/api/", - json_payload={ - "email": self.email, - "password": self.password, - }, - include_auth=False, - ) + async def _authenticate(self, previous_token: str | None) -> str: + """Refresh or log in once, even when several requests receive a 401.""" + async with self._auth_lock: + if self.access_token is not None and self.access_token != previous_token: + return self.access_token + if self.use_refresh_tokens and self.refresh_token: + response, status = await self.request( + uri="refreshAuthToken", + path="/api/", + json_payload={"refresh-token": self.refresh_token}, + include_auth=False, + pass_through_statuses=frozenset({401, 503}), + retry_on_timeout=False, + ) + if status == 401: + self.refresh_token = None + elif status == 503: + raise ConnectionError( + "Refresh failed with HTTP 503; its outcome may be unknown." + ) + else: + return self._store_token_response(response, require_refresh=True) + return await self._login() + + def _store_token_response( + self, response: dict | list, *, require_refresh: bool = False + ) -> str: if not isinstance(response, dict): raise ValueError( - f"Expected a JSON object from requestAuthToken, but got a {type(response).__name__}." + "Expected a JSON object from requestAuthToken, but got a " + f"{type(response).__name__}." ) - access_token: str = response["auth_token"] + access_token = response["auth_token"] + refresh_token = response.get("refresh-token") + if require_refresh and not isinstance(refresh_token, str): + raise ValueError("Refresh response did not contain a refresh token.") self.access_token = access_token + self.refresh_token = refresh_token + if refresh_token is not None and self.refresh_token_callback is not None: + self.refresh_token_callback(refresh_token) return access_token + async def get_access_token(self) -> str: + """Refresh or log in, then store and return the access token.""" + return await self._authenticate(self.access_token) + + async def _login(self) -> str: + """Log in with credentials and store and return the access token.""" + if self.email is None or self.password is None: + if not sys.stdin.isatty(): + raise RuntimeError( + "Email and password are required for login; no interactive terminal is available." + ) + if self.email is None: + email = await asyncio.to_thread(input, "FlexMeasures email: ") + if not re.match(r".+\@.+\..+", email): + raise EmailValidationError( + f"{email} is not an email address format string" + ) + self.email = email + if self.password is None: + password = await asyncio.to_thread( + getpass.getpass, "FlexMeasures password: " + ) + if not password: + raise EmptyPasswordError("password cannot be empty") + self.password = password + payload: dict[str, Any] = {"email": self.email, "password": self.password} + if self.use_refresh_tokens: + payload["include-refresh-token"] = True + response, status = await self.request( + uri="requestAuthToken", + path="/api/", + json_payload=payload, + include_auth=False, + pass_through_statuses=( + frozenset({422}) if self.use_refresh_tokens else frozenset() + ), + ) + # Older servers reject unknown fields. Only retry for that specific error. + if status == 422: + message = ( + str(response.get("message", "")) if isinstance(response, dict) else "" + ) + if "include-refresh-token" not in message or "Unknown field" not in message: + raise ValueError(f"Request failed with status code 422: {message}") + del payload["include-refresh-token"] + response, _ = await self.request( + uri="requestAuthToken", + path="/api/", + json_payload=payload, + include_auth=False, + ) + return self._store_token_response(response) + async def get_versions(self): """Get the FlexMeasures version and the supported API versions of the FlexMeasures server. diff --git a/src/flexmeasures_client/response_handling.py b/src/flexmeasures_client/response_handling.py index 1f22ae3f..7f04cc37 100644 --- a/src/flexmeasures_client/response_handling.py +++ b/src/flexmeasures_client/response_handling.py @@ -59,6 +59,7 @@ async def check_response( method: str = "GET", pass_through_statuses: frozenset[int] = frozenset(), rate_limit_callback: Callable[[str], None] | None = None, + auth_token_used: str | None = None, ) -> tuple[int, bool, URL]: """ <300: passes @@ -116,7 +117,7 @@ async def check_response( payload: {payload}. Re-authenticating!""" self.logger.debug(message) - await self.get_access_token() + await self._authenticate(auth_token_used) reauth_once = False elif status == 429 or (status == 503 and "Retry-After" in headers): sleep_interval = parse_retry_after(headers.get("Retry-After")) diff --git a/tests/client/test_init.py b/tests/client/test_init.py index a2b14436..3e2c8bc5 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -86,6 +86,9 @@ async def test__init__( "password": password, "email": email, "access_token": None, + "use_refresh_tokens": True, + "refresh_token": None, + "refresh_token_callback": None, "host": asserted_host, "port": asserted_port, "scheme": asserted_scheme, @@ -111,6 +114,7 @@ async def test__init__( init_dict.pop("session") init_dict.pop("logger") init_dict.pop("_sensor_asset_id_cache") + init_dict.pop("_auth_lock") assert init_dict == assert_dict @@ -276,7 +280,11 @@ async def test_get_access_token() -> None: m.assert_called_once_with( "http://localhost:5000/api/requestAuthToken", method="POST", - json={"email": "test@test.test", "password": "test"}, + json={ + "email": "test@test.test", + "password": "test", + "include-refresh-token": True, + }, headers={ "Content-Type": "application/json", CLIENT_VERSION_HEADER: __version__, diff --git a/tests/client/test_refresh_tokens.py b/tests/client/test_refresh_tokens.py new file mode 100644 index 00000000..c426e682 --- /dev/null +++ b/tests/client/test_refresh_tokens.py @@ -0,0 +1,272 @@ +import asyncio +import logging + +import pytest +from aioresponses import aioresponses + +from flexmeasures_client import FlexMeasuresClient + +BASE = "http://localhost:5000/api/" + + +async def _run_prompt(func, *args): + return func(*args) + + +@pytest.mark.asyncio +async def test_login_and_refresh_rotate_and_notify(caplog): + saved = [] + with aioresponses() as mock: + mock.post( + BASE + "requestAuthToken", + payload={"auth_token": "access1", "refresh-token": "refresh1"}, + ) + mock.get(BASE + "v3_0/sensors", status=401, payload={}) + mock.post( + BASE + "refreshAuthToken", + payload={"auth_token": "access2", "refresh-token": "refresh2"}, + ) + mock.get(BASE + "v3_0/sensors", payload=[]) + client = FlexMeasuresClient( + "secret-password", + "test@test.test", + use_refresh_tokens=True, + refresh_token_callback=saved.append, + ) + with caplog.at_level(logging.DEBUG): + await client.get_sensors(parse_json_fields=False) + assert saved == ["refresh1", "refresh2"] + assert client.access_token == "access2" + assert client.refresh_token == "refresh2" + assert mock.requests[ + ("POST", client.build_url("refreshAuthToken", path="/api/")) + ][0].kwargs["json"] == {"refresh-token": "refresh1"} + assert all( + secret not in caplog.text + for secret in [ + "secret-password", + "access1", + "access2", + "refresh1", + "refresh2", + ] + ) + assert "secret-password" not in repr(client) and "refresh2" not in repr(client) + await client.close() + + +@pytest.mark.asyncio +async def test_old_server_unknown_field_falls_back(): + with aioresponses() as mock: + mock.post( + BASE + "requestAuthToken", + status=422, + payload={ + "message": {"json": {"include-refresh-token": ["Unknown field."]}} + }, + ) + mock.post(BASE + "requestAuthToken", payload={"auth_token": "legacy"}) + client = FlexMeasuresClient( + "password", "test@test.test", use_refresh_tokens=True + ) + assert await client.get_access_token() == "legacy" + assert client.refresh_token is None + calls = mock.requests[ + ("POST", client.build_url("requestAuthToken", path="/api/")) + ] + assert calls[0].kwargs["json"] == { + "email": client.email, + "password": client.password, + "include-refresh-token": True, + } + assert calls[1].kwargs["json"] == { + "email": client.email, + "password": client.password, + } + await client.close() + + +@pytest.mark.asyncio +async def test_rejected_refresh_logs_in_again(): + with aioresponses() as mock: + mock.post(BASE + "refreshAuthToken", status=401, payload={}) + mock.post( + BASE + "requestAuthToken", + payload={"auth_token": "new", "refresh-token": "new-refresh"}, + ) + client = FlexMeasuresClient( + "password", + "test@test.test", + use_refresh_tokens=True, + refresh_token="expired", + ) + assert (await client.get_headers(True))["Authorization"] == "new" + assert client.refresh_token == "new-refresh" + await client.close() + + +@pytest.mark.asyncio +async def test_concurrent_reauthentication_rotates_only_once(): + with aioresponses() as mock: + mock.post( + BASE + "refreshAuthToken", + payload={"auth_token": "new", "refresh-token": "rotated"}, + ) + client = FlexMeasuresClient( + "password", + "test@test.test", + use_refresh_tokens=True, + access_token="old", + refresh_token="original", + ) + assert await asyncio.gather( + client._authenticate("old"), client._authenticate("old") + ) == ["new", "new"] + assert client.refresh_token == "rotated" + mock.assert_called_once() + await client.close() + + +@pytest.mark.asyncio +async def test_refresh_503_does_not_retry_or_send_password(): + with aioresponses() as mock: + mock.post(BASE + "refreshAuthToken", status=503, payload={}) + client = FlexMeasuresClient( + "password", + "test@test.test", + use_refresh_tokens=True, + refresh_token="original", + ) + with pytest.raises(ConnectionError, match="503"): + await client.get_headers(True) + mock.assert_called_once() + await client.close() + + +@pytest.mark.asyncio +async def test_refresh_timeout_does_not_retry(): + with aioresponses() as mock: + mock.post(BASE + "refreshAuthToken", exception=asyncio.TimeoutError()) + client = FlexMeasuresClient( + "password", + "test@test.test", + use_refresh_tokens=True, + refresh_token="original", + ) + with pytest.raises(ConnectionError, match="outcome is unknown"): + await client.get_headers(True) + mock.assert_called_once() + await client.close() + + +@pytest.mark.asyncio +async def test_unrelated_login_validation_error_does_not_fall_back(): + with aioresponses() as mock: + mock.post( + BASE + "requestAuthToken", + status=422, + payload={"message": {"email": ["Invalid email"]}}, + ) + client = FlexMeasuresClient( + "password", "test@test.test", use_refresh_tokens=True + ) + with pytest.raises(ValueError, match="Invalid email"): + await client.get_access_token() + mock.assert_called_once() + await client.close() + + +@pytest.mark.asyncio +async def test_get_access_token_refreshes_without_sending_password(): + with aioresponses() as mock: + mock.post( + BASE + "refreshAuthToken", + payload={"auth_token": "new", "refresh-token": "rotated"}, + ) + client = FlexMeasuresClient( + "password", "test@test.test", refresh_token="original" + ) + assert await client.get_access_token() == "new" + assert client.refresh_token == "rotated" + mock.assert_called_once() + await client.close() + + +@pytest.mark.asyncio +async def test_refresh_tokens_can_be_disabled(): + with aioresponses() as mock: + mock.post(BASE + "requestAuthToken", payload={"auth_token": "new"}) + client = FlexMeasuresClient( + "password", "test@test.test", use_refresh_tokens=False + ) + assert await client.get_access_token() == "new" + calls = mock.requests[ + ("POST", client.build_url("requestAuthToken", path="/api/")) + ] + assert calls[0].kwargs["json"] == { + "email": client.email, + "password": client.password, + } + await client.close() + + +@pytest.mark.asyncio +async def test_token_only_authentication(): + with aioresponses() as mock: + mock.post( + BASE + "refreshAuthToken", + payload={"auth_token": "new", "refresh-token": "rotated"}, + ) + client = FlexMeasuresClient(refresh_token="original") + assert await client.get_access_token() == "new" + mock.assert_called_once() + await client.close() + + +@pytest.mark.asyncio +async def test_interactive_login_and_rejected_token(monkeypatch): + monkeypatch.setattr(asyncio, "to_thread", _run_prompt) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("builtins.input", lambda prompt: "user@example.com") + monkeypatch.setattr("getpass.getpass", lambda prompt: "secret") + with aioresponses() as mock: + mock.post(BASE + "refreshAuthToken", status=401, payload={}) + mock.post( + BASE + "requestAuthToken", + payload={"auth_token": "new", "refresh-token": "rotated"}, + ) + client = FlexMeasuresClient(refresh_token="expired") + assert await client.get_access_token() == "new" + calls = mock.requests[ + ("POST", client.build_url("requestAuthToken", path="/api/")) + ] + assert calls[0].kwargs["json"] == { + "email": "user@example.com", + "password": "secret", + "include-refresh-token": True, + } + await client.close() + + +@pytest.mark.asyncio +async def test_noninteractive_login_requires_credentials(monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: False) + client = FlexMeasuresClient() + with pytest.raises(RuntimeError, match="no interactive terminal"): + await client.get_access_token() + await client.close() + + +@pytest.mark.asyncio +async def test_interactive_prompts_only_for_missing_password(monkeypatch): + monkeypatch.setattr(asyncio, "to_thread", _run_prompt) + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "builtins.input", lambda prompt: pytest.fail("Unexpected email prompt") + ) + monkeypatch.setattr("getpass.getpass", lambda prompt: "secret") + with aioresponses() as mock: + mock.post(BASE + "requestAuthToken", payload={"auth_token": "new"}) + client = FlexMeasuresClient(email="user@example.com") + assert await client.get_access_token() == "new" + await client.close()