diff --git a/.github/workflows/dependency-testing.yaml b/.github/workflows/dependency-testing.yaml index b07ca4c9f..bca85b137 100644 --- a/.github/workflows/dependency-testing.yaml +++ b/.github/workflows/dependency-testing.yaml @@ -5,6 +5,7 @@ on: branches: [main] paths: - 'pyproject.toml' + - 'packages/**' - 'requirements/**' - 'validmind/**' - 'tests/**' @@ -12,6 +13,7 @@ on: branches: ['*'] paths: - 'pyproject.toml' + - 'packages/**' - 'requirements/**' - 'validmind/**' - 'tests/**' @@ -68,9 +70,9 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - name: Build wheel and sdist + - name: Build wheels and sdists run: | - python -m build + uv build --all-packages - name: Install and test via pip artifacts (${{ matrix.deps-type }}) run: | @@ -82,7 +84,9 @@ jobs: if [[ "${{ matrix.deps-type }}" == "core" ]]; then # Exercise the unbounded core dependency set separately from the # constrained optional extras (for example, latest scikit-learn). - WHEEL=$(ls dist/*.whl | head -n 1) + WHEEL=$(ls dist/validmind-*.whl | head -n 1) + CORE_WHEEL=$(ls dist/validmind_tracking_core*.whl | head -n 1) + python -m pip install "${CORE_WHEEL}" # Jinja2 is currently imported by the library but is only present # transitively through optional extras. Install it as test tooling # so this leg can exercise the otherwise unbounded core set. @@ -95,7 +99,9 @@ jobs: | cat elif [[ "${{ matrix.deps-type }}" == "default" ]]; then # Install only from built artifacts (let pip resolve deps normally) - WHEEL=$(ls dist/*.whl | head -n 1) + WHEEL=$(ls dist/validmind-*.whl | head -n 1) + CORE_WHEEL=$(ls dist/validmind_tracking_core*.whl | head -n 1) + python -m pip install "${CORE_WHEEL}" python -m pip install "${WHEEL}[all]" pip check python -m tests.test_unit_tests | cat @@ -105,7 +111,9 @@ jobs: uv pip compile pyproject.toml -p "${{ matrix.python-version }}" --all-extras --no-emit-index-url --no-annotate --no-strip-markers --output-file "$OUT" --upgrade # Install constraints then the wheel without reinstalling deps pip install -r "$OUT" - WHEEL=$(ls dist/*.whl | head -n 1) + WHEEL=$(ls dist/validmind-*.whl | head -n 1) + CORE_WHEEL=$(ls dist/validmind_tracking_core*.whl | head -n 1) + pip install "${CORE_WHEEL}" --no-deps python -m pip install "${WHEEL}[all]" --no-deps pip check python -m tests.test_unit_tests | cat @@ -146,6 +154,7 @@ jobs: fi python -m pip install --upgrade pip pip install -r "$FREEZE_FILE" + python -m pip install packages/validmind-tracking-core --no-deps python -m pip install . --no-deps python -m tests.test_unit_tests | cat diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 347733f6d..c3f3422de 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -44,7 +44,7 @@ jobs: sudo apt install r-base r-base-dev - name: Build the package - run: uv build + run: uv build --all-packages - name: Remove Build Environment run: rm -rf .venv @@ -52,9 +52,11 @@ jobs: - name: 'Setup Virtual Environment for [all]' run: python -m venv all-venv - # This proves that the [all] install target works + # This proves that the [all] install target works - name: 'Install Built Package for [all]' - run: all-venv/bin/pip install --no-cache-dir "$(ls dist/validmind*.whl | head -n 1)[all]" + run: | + all-venv/bin/pip install --no-cache-dir "$(ls dist/validmind_tracking_core*.whl | head -n 1)" + all-venv/bin/pip install --no-cache-dir "$(ls dist/validmind-*.whl | head -n 1)[all]" - name: Install Additional Dependencies run: all-venv/bin/pip install nbformat papermill jupyter diff --git a/.github/workflows/pypi-metrics.yaml b/.github/workflows/pypi-metrics.yaml new file mode 100644 index 000000000..5088b201e --- /dev/null +++ b/.github/workflows/pypi-metrics.yaml @@ -0,0 +1,37 @@ +# Publish the lightweight metric SDK independently from the full library. + +name: Publish Metrics SDK to PyPI + +on: + push: + tags: + - 'metrics-v*.*.*' + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Build metrics SDK + run: uv build --package validmind-metrics --out-dir dist + + - name: Publish metrics SDK + env: + UV_PUBLISH_USERNAME: __token__ + UV_PUBLISH_PASSWORD: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} + run: uv publish dist/validmind_metrics-* diff --git a/.github/workflows/pypi-tracking-core.yaml b/.github/workflows/pypi-tracking-core.yaml new file mode 100644 index 000000000..9b31f94ce --- /dev/null +++ b/.github/workflows/pypi-tracking-core.yaml @@ -0,0 +1,37 @@ +# Publish the dependency-light tracking core independently from the full library. + +name: Publish Tracking Core to PyPI + +on: + push: + tags: + - 'tracking-core-v*.*.*' + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Build tracking core + run: uv build --package validmind-tracking-core --out-dir dist + + - name: Publish tracking core + env: + UV_PUBLISH_USERNAME: __token__ + UV_PUBLISH_PASSWORD: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} + run: uv publish dist/validmind_tracking_core-* diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 96b5bf2c5..37e45f2c8 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -1,4 +1,5 @@ -# This workflow pushes the ValidMind Library package to PyPI when a tag is created +# This workflow pushes the ValidMind Library package to PyPI when a tag is created. +# The validmind-tracking-core package must be published first for new releases. name: Publish to PyPI diff --git a/Makefile b/Makefile index 2c64e0b94..d612eaeac 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,13 @@ __check_defined = \ $(error Undefined $1$(if $2, ($2)))) format: - uv run black validmind - uv run isort validmind + uv run black validmind packages + uv run isort validmind packages lint: # don't check max line length for now since black already takes care of it # and flake8 is too strict where it doesn't need to be - uv run flake8 validmind --config .flake8 + uv run flake8 validmind packages --config .flake8 install: uv sync --all-extras --group dev @@ -28,8 +28,13 @@ ifdef ONLY uv run python -m unittest $(ONLY) else uv run python -m unittest discover tests + $(MAKE) test-packages endif +test-packages: + uv run --package validmind-tracking-core python -m unittest discover packages/validmind-tracking-core/tests + uv run --package validmind-metrics python -m unittest discover packages/validmind-metrics/tests + test-unit: uv run python -m unittest "tests.test_unit_tests" diff --git a/packages/validmind-metrics/README.md b/packages/validmind-metrics/README.md new file mode 100644 index 000000000..e5f001899 --- /dev/null +++ b/packages/validmind-metrics/README.md @@ -0,0 +1,25 @@ +# ValidMind Metrics + +`validmind-metrics` is a lightweight client for sending unit metrics to the +ValidMind Platform. It supports API-key authentication and OIDC device-flow +authentication without installing or importing the full `validmind` library. + +```python +from validmind_metrics import MetricsClient + +client = MetricsClient( + api_host="https://app.validmind.ai/api/v1/tracking", + model="model-cuid", + api_key="api-key", + api_secret="api-secret", +) + +client.log_metric("accuracy", 0.95) +``` + +OIDC authentication uses the same `issuer`, `client_id`, optional `scope`, and +optional `audience` settings as the full library. Cached credentials are stored +under `~/.validmind/credentials.json`. + +For an async HTTP handler, use `await client.alog_metric(...)` so the blocking +transport runs outside the event loop. diff --git a/packages/validmind-metrics/pyproject.toml b/packages/validmind-metrics/pyproject.toml new file mode 100644 index 000000000..ab6a588f7 --- /dev/null +++ b/packages/validmind-metrics/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["hatchling>=1.26.0"] +build-backend = "hatchling.build" + +[project] +name = "validmind-metrics" +version = "0.1.0" +description = "Lightweight metric logging client for the ValidMind Platform" +readme = "README.md" +requires-python = ">=3.9,<3.15" +license = { text = "AGPL-3.0 AND ValidMind Commercial" } +dependencies = [ + "validmind-tracking-core>=0.1.0,<0.2.0", +] + +[tool.uv.sources] +validmind-tracking-core = { workspace = true } + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/README.md", + "/pyproject.toml", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/validmind_metrics"] diff --git a/packages/validmind-metrics/src/validmind_metrics/__init__.py b/packages/validmind-metrics/src/validmind_metrics/__init__.py new file mode 100644 index 000000000..5c9b5ea76 --- /dev/null +++ b/packages/validmind-metrics/src/validmind_metrics/__init__.py @@ -0,0 +1,85 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Lightweight metric logging client for the ValidMind Platform.""" + +from typing import Any, Dict, List, Optional + +from validmind_tracking_core import ( + MetricsClient, + TrackingAPIError, + TrackingAuthError, + TrackingConfigurationError, +) + +__version__ = "0.1.0" + +_client: Optional[MetricsClient] = None + + +def init(**kwargs: Any) -> MetricsClient: + """Create and retain the default metric client.""" + global _client + kwargs.setdefault("client_version", __version__) + _client = MetricsClient(**kwargs) + return _client + + +def _get_client() -> MetricsClient: + if _client is None: + return init() + return _client + + +def log_metric( + key: str, + value: Any, + inputs: Optional[List[str]] = None, + params: Optional[Dict[str, Any]] = None, + recorded_at: Optional[str] = None, + thresholds: Optional[Dict[str, Any]] = None, + passed: Optional[bool] = None, +) -> Dict[str, Any]: + """Log one metric using the default client.""" + return _get_client().log_metric( + key, + value, + inputs=inputs, + params=params, + recorded_at=recorded_at, + thresholds=thresholds, + passed=passed, + ) + + +async def alog_metric( + key: str, + value: Any, + inputs: Optional[List[str]] = None, + params: Optional[Dict[str, Any]] = None, + recorded_at: Optional[str] = None, + thresholds: Optional[Dict[str, Any]] = None, + passed: Optional[bool] = None, +) -> Dict[str, Any]: + """Log one metric without blocking the current event loop.""" + return await _get_client().alog_metric( + key, + value, + inputs=inputs, + params=params, + recorded_at=recorded_at, + thresholds=thresholds, + passed=passed, + ) + + +__all__ = [ + "MetricsClient", + "TrackingAPIError", + "TrackingAuthError", + "TrackingConfigurationError", + "alog_metric", + "init", + "log_metric", +] diff --git a/packages/validmind-metrics/tests/test_metrics.py b/packages/validmind-metrics/tests/test_metrics.py new file mode 100644 index 000000000..60d32dc9a --- /dev/null +++ b/packages/validmind-metrics/tests/test_metrics.py @@ -0,0 +1,144 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +import asyncio +import json +import subprocess +import sys +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from validmind_metrics import MetricsClient +from validmind_tracking_core.credentials_store import upsert_cached_entry + + +class TestMetricsClient(unittest.TestCase): + def _response(self, body=None, status_code=200, text=""): + response = Mock() + response.status_code = status_code + response.text = text + response.json.return_value = body or {"ok": True} + return response + + @patch("validmind_tracking_core.metrics.requests.post") + def test_log_metric_uses_api_key_headers_and_schema(self, mock_post): + mock_post.return_value = self._response({"metric_id": "metric-1"}) + client = MetricsClient( + api_host="https://tracking.example/api/v1/tracking", + model="model-1", + api_key="key", + api_secret="secret", + monitoring=True, + document="monitoring", + client_version="test-client/1.0", + ) + + result = client.log_metric( + "accuracy", + 0.95, + inputs=["dataset-1"], + params={"average": "macro"}, + recorded_at="2026-08-27T00:00:00Z", + thresholds={"minimum": 0.9}, + passed=True, + ) + + self.assertEqual(result, {"metric_id": "metric-1"}) + mock_post.assert_called_once() + url = mock_post.call_args.args[0] + kwargs = mock_post.call_args.kwargs + self.assertEqual( + url, "https://tracking.example/api/v1/tracking/log_unit_metric" + ) + self.assertEqual( + kwargs["headers"], + { + "X-MODEL-CUID": "model-1", + "X-MONITORING": "True", + "X-LIBRARY-VERSION": "test-client/1.0", + "X-DOCUMENT-TYPE": "monitoring", + "X-API-KEY": "key", + "X-API-SECRET": "secret", + }, + ) + self.assertEqual( + json.loads(kwargs["data"]), + { + "key": "accuracy", + "value": 0.95, + "inputs": ["dataset-1"], + "params": {"average": "macro"}, + "recorded_at": "2026-08-27T00:00:00Z", + "thresholds": {"minimum": 0.9}, + "passed": True, + }, + ) + + @patch("validmind_tracking_core.metrics.requests.post") + def test_cached_oidc_token_is_used(self, mock_post): + mock_post.return_value = self._response({"metric_id": "metric-2"}) + with TemporaryDirectory() as temp_dir: + credentials_path = Path(temp_dir) / "credentials.json" + upsert_cached_entry( + "https://issuer.example", + "client-1", + { + "access_token": "cached-token", + "refresh_token": "refresh-token", + "expires_at": ( + datetime.now(timezone.utc) + timedelta(hours=1) + ).isoformat(), + }, + path=credentials_path, + ) + + client = MetricsClient( + api_host="https://tracking.example/api/v1/tracking", + model="model-1", + issuer="https://issuer.example/", + client_id="client-1", + credentials_path=credentials_path, + ) + client.log_metric("accuracy", 0.95) + + headers = mock_post.call_args.kwargs["headers"] + self.assertEqual(headers["Authorization"], "Bearer cached-token") + self.assertNotIn("X-API-KEY", headers) + + @patch("validmind_tracking_core.metrics.requests.post") + def test_async_metric_does_not_require_nested_event_loop(self, mock_post): + mock_post.return_value = self._response({"ok": True}) + client = MetricsClient( + api_host="https://tracking.example/api/v1/tracking", + model="model-1", + api_key="key", + api_secret="secret", + ) + + async def handler(): + return await client.alog_metric("accuracy", 0.95) + + self.assertEqual(asyncio.run(handler()), {"ok": True}) + + def test_import_isolated_from_full_library(self): + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import validmind_metrics; " + "assert 'validmind' not in sys.modules; " + "assert 'aiohttp' not in sys.modules", + ], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(result.stderr, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/validmind-tracking-core/README.md b/packages/validmind-tracking-core/README.md new file mode 100644 index 000000000..f90868284 --- /dev/null +++ b/packages/validmind-tracking-core/README.md @@ -0,0 +1,8 @@ +# ValidMind Tracking Core + +`validmind-tracking-core` contains the dependency-light authentication and tracking +transport shared by ValidMind SDKs. It supports API-key authentication and OIDC +device-flow authentication without importing the full `validmind` package. + +This package is an implementation dependency of ValidMind SDKs. Application code +should normally use `validmind-metrics` or `validmind` directly. diff --git a/packages/validmind-tracking-core/pyproject.toml b/packages/validmind-tracking-core/pyproject.toml new file mode 100644 index 000000000..d23742524 --- /dev/null +++ b/packages/validmind-tracking-core/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["hatchling>=1.26.0"] +build-backend = "hatchling.build" + +[project] +name = "validmind-tracking-core" +version = "0.1.0" +description = "Dependency-light authentication and tracking transport for ValidMind SDKs" +readme = "README.md" +requires-python = ">=3.9,<3.15" +license = { text = "AGPL-3.0 AND ValidMind Commercial" } +dependencies = [ + "requests (>=2.28.0,<3.0.0)", +] + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/README.md", + "/pyproject.toml", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/validmind_tracking_core"] diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/__init__.py b/packages/validmind-tracking-core/src/validmind_tracking_core/__init__.py new file mode 100644 index 000000000..88cf0dda8 --- /dev/null +++ b/packages/validmind-tracking-core/src/validmind_tracking_core/__init__.py @@ -0,0 +1,17 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Dependency-light authentication and tracking primitives for ValidMind SDKs.""" + +from .errors import TrackingAPIError, TrackingAuthError, TrackingConfigurationError +from .metrics import MetricsClient, post_metric, serialize_metric + +__all__ = [ + "MetricsClient", + "TrackingAPIError", + "TrackingAuthError", + "TrackingConfigurationError", + "post_metric", + "serialize_metric", +] diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/credentials_store.py b/packages/validmind-tracking-core/src/validmind_tracking_core/credentials_store.py new file mode 100644 index 000000000..fde7b2d75 --- /dev/null +++ b/packages/validmind-tracking-core/src/validmind_tracking_core/credentials_store.py @@ -0,0 +1,163 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Small, file-backed OIDC credential store used by the tracking SDKs.""" + +from __future__ import annotations + +import json +import os +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +from .errors import TrackingAuthError + +_CREDENTIALS_VERSION = 1 + + +def normalize_issuer(issuer: str) -> str: + base = issuer.strip().rstrip("/") + while len(base) >= 2 and base[0] == base[-1] and base[0] in ('"', "'"): + base = base[1:-1].strip().rstrip("/") + return base + + +def normalize_client_id(client_id: str) -> str: + base = client_id.strip() + while len(base) >= 2 and base[0] == base[-1] and base[0] in ('"', "'"): + base = base[1:-1].strip() + return base + + +def normalize_audience(audience: Optional[str]) -> str: + if not audience: + return "" + base = audience.strip() + while len(base) >= 2 and base[0] == base[-1] and base[0] in ('"', "'"): + base = base[1:-1].strip() + return base + + +def credential_key(issuer: str, client_id: str, audience: Optional[str] = None) -> str: + base = f"{normalize_issuer(issuer)}|{normalize_client_id(client_id)}" + aud = normalize_audience(audience) + return f"{base}|{aud}" if aud else base + + +def credentials_path() -> Path: + return Path.home() / ".validmind" / "credentials.json" + + +def _empty_store() -> Dict[str, Any]: + return {"version": _CREDENTIALS_VERSION, "credentials": {}} + + +def load_credentials_file(path: Optional[Path] = None) -> Dict[str, Any]: + path = path or credentials_path() + if not path.is_file(): + return _empty_store() + try: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + except (json.JSONDecodeError, OSError) as exc: + raise TrackingAuthError( + f"Could not read credentials file {path}: {exc}" + ) from exc + if not isinstance(data, dict): + raise TrackingAuthError(f"Invalid credentials file format at {path}") + data.setdefault("version", _CREDENTIALS_VERSION) + data.setdefault("credentials", {}) + return data + + +def _atomic_write(path: Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp( + dir=str(path.parent), prefix=".credentials-", suffix=".tmp", text=True + ) + temp_path = Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) + os.chmod(temp_path, 0o600) + os.replace(temp_path, path) + except Exception: + try: + temp_path.unlink() + except OSError: + pass + raise + + +def save_credentials_file(data: Dict[str, Any], path: Optional[Path] = None) -> None: + path = path or credentials_path() + normalized = dict(data) + normalized["version"] = _CREDENTIALS_VERSION + if not isinstance(normalized.get("credentials"), dict): + normalized["credentials"] = {} + _atomic_write(path, normalized) + + +def get_cached_entry( + issuer: str, + client_id: str, + path: Optional[Path] = None, + audience: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + key = credential_key(issuer, client_id, audience) + entry = load_credentials_file(path).get("credentials", {}).get(key) + return dict(entry) if entry else None + + +def upsert_cached_entry( + issuer: str, + client_id: str, + entry: Dict[str, Any], + path: Optional[Path] = None, + audience: Optional[str] = None, +) -> None: + key = credential_key(issuer, client_id, audience) + normalized_issuer = normalize_issuer(issuer) + normalized_audience = normalize_audience(audience) + data = load_credentials_file(path) + credentials = dict(data.get("credentials", {})) + row = {"issuer": normalized_issuer, "client_id": client_id, **entry} + if normalized_audience: + row["audience"] = normalized_audience + credentials[key] = row + data["credentials"] = credentials + save_credentials_file(data, path) + + +def delete_cached_entry( + issuer: str, + client_id: str, + path: Optional[Path] = None, + audience: Optional[str] = None, +) -> None: + data = load_credentials_file(path) + credentials = dict(data.get("credentials", {})) + credentials.pop(credential_key(issuer, client_id, audience), None) + data["credentials"] = credentials + save_credentials_file(data, path) + + +def is_expired(entry: Dict[str, Any], skew_seconds: int = 120) -> bool: + raw = entry.get("expires_at") + if not raw: + return True + try: + expires = datetime.fromisoformat(raw.replace("Z", "+00:00")) + except ValueError: + return True + if expires.tzinfo is None: + expires = expires.replace(tzinfo=timezone.utc) + return datetime.now(timezone.utc) >= expires - timedelta(seconds=skew_seconds) + + +def expires_at_from_secs(expires_in: Optional[int]) -> str: + seconds = int(expires_in) if expires_in is not None else 3600 + return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat() diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/errors.py b/packages/validmind-tracking-core/src/validmind_tracking_core/errors.py new file mode 100644 index 000000000..242375e07 --- /dev/null +++ b/packages/validmind-tracking-core/src/validmind_tracking_core/errors.py @@ -0,0 +1,26 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Errors raised by the dependency-light tracking core.""" + + +class TrackingError(Exception): + """Base class for tracking-core failures.""" + + +class TrackingConfigurationError(TrackingError): + """The tracking client configuration is invalid or incomplete.""" + + +class TrackingAuthError(TrackingError): + """API-key or OIDC authentication failed.""" + + +class TrackingAPIError(TrackingError): + """The tracking API rejected a request or returned an invalid response.""" + + def __init__(self, status_code: int, message: str, response_text: str = ""): + super().__init__(message) + self.status_code = status_code + self.response_text = response_text diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/metrics.py b/packages/validmind-tracking-core/src/validmind_tracking_core/metrics.py new file mode 100644 index 000000000..fba49ff2b --- /dev/null +++ b/packages/validmind-tracking-core/src/validmind_tracking_core/metrics.py @@ -0,0 +1,213 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Dependency-light synchronous and async-compatible metric transport.""" + +from __future__ import annotations + +import asyncio +import json +import os +from typing import Any, Dict, List, Optional, Type +from urllib.parse import urljoin + +import requests + +from .errors import TrackingAPIError, TrackingConfigurationError +from .oidc import OIDCAuthenticator + + +def _validate_metric( + key: str, + value: Any, + thresholds: Optional[Dict[str, Any]], +) -> None: + if not key or not isinstance(key, str): + raise ValueError("`key` must be a non-empty string") + if value is None: + raise ValueError("Must provide a value for the metric") + if not isinstance(value, (int, float)): + raise ValueError( + "Only scalar values (int or float) are allowed for logging metrics." + ) + if thresholds is not None and not isinstance(thresholds, dict): + raise ValueError("`thresholds` must be a dictionary or None") + + +def serialize_metric( + key: str, + value: Any, + inputs: Optional[List[str]] = None, + params: Optional[Dict[str, Any]] = None, + recorded_at: Optional[str] = None, + thresholds: Optional[Dict[str, Any]] = None, + passed: Optional[bool] = None, + *, + encoder: Optional[Type[json.JSONEncoder]] = None, +) -> str: + """Validate and serialize a metric using the tracking API schema.""" + _validate_metric(key, value, thresholds) + payload = { + "key": key, + "value": value, + "inputs": inputs or [], + "params": params or {}, + "recorded_at": recorded_at, + "thresholds": thresholds or {}, + "passed": passed if passed is not None else None, + } + kwargs = {"allow_nan": False} + if encoder is not None: + kwargs["cls"] = encoder + return json.dumps(payload, **kwargs) + + +def post_metric( + url: str, + body: str, + headers: Dict[str, str], + *, + timeout: float = 30.0, +) -> Dict[str, Any]: + """POST a serialized metric and return the JSON response.""" + response = requests.post(url, data=body, headers=headers, timeout=timeout) + if response.status_code != 200: + raise TrackingAPIError(response.status_code, response.text[:500], response.text) + try: + result = response.json() + except ValueError as exc: + raise TrackingAPIError( + response.status_code, + "ValidMind returned a non-JSON metric response", + response.text, + ) from exc + if not isinstance(result, dict): + raise TrackingAPIError( + response.status_code, + "ValidMind returned an invalid metric response", + response.text, + ) + return result + + +class MetricsClient: + """Client for the ValidMind ``log_unit_metric`` endpoint.""" + + def __init__( + self, + *, + api_host: Optional[str] = None, + api_url: Optional[str] = None, + model: Optional[str] = None, + api_key: Optional[str] = None, + api_secret: Optional[str] = None, + monitoring: bool = False, + document: Optional[str] = None, + client_version: str = "validmind-tracking-core/0.1.0", + timeout: Optional[float] = None, + issuer: Optional[str] = None, + client_id: Optional[str] = None, + scope: Optional[str] = None, + audience: Optional[str] = None, + credentials_path=None, + status_callback=None, + ): + self.api_host = ( + api_url or api_host or os.getenv("VM_API_URL") or os.getenv("VM_API_HOST") + ) + self.model = model or os.getenv("VM_API_MODEL") + self.monitoring = monitoring + self.document = document + self.client_version = client_version + self.timeout = float(timeout or os.getenv("VM_API_TIMEOUT", 30)) + self._api_key = api_key if api_key is not None else os.getenv("VM_API_KEY") + self._api_secret = ( + api_secret if api_secret is not None else os.getenv("VM_API_SECRET") + ) + issuer = issuer if issuer is not None else os.getenv("VM_OIDC_ISSUER") + client_id = ( + client_id if client_id is not None else os.getenv("VM_OIDC_CLIENT_ID") + ) + scope = scope if scope is not None else os.getenv("VM_OIDC_SCOPE") + audience = audience if audience is not None else os.getenv("VM_OIDC_AUDIENCE") + + has_api_creds = bool(self._api_key and self._api_secret) + has_oidc = bool(issuer and client_id) + if not self.api_host: + raise TrackingConfigurationError("API host must be provided") + if not self.model: + raise TrackingConfigurationError("Model ID must be provided") + if has_api_creds and has_oidc: + raise TrackingConfigurationError( + "Provide either API credentials or OIDC credentials, not both" + ) + if bool(issuer) != bool(client_id): + raise TrackingConfigurationError( + "issuer and client_id must be provided together" + ) + if not has_api_creds and not has_oidc: + raise TrackingConfigurationError( + "Provide API credentials or issuer and client_id for OIDC" + ) + + self._oidc = None + if has_oidc: + self._oidc = OIDCAuthenticator( + issuer, + client_id, + scope=scope, + audience=audience, + timeout=self.timeout, + credentials_path_value=credentials_path, + status_callback=status_callback, + ) + self._oidc.initialize() + + def log_metric( + self, + key: str, + value: Any, + inputs: Optional[List[str]] = None, + params: Optional[Dict[str, Any]] = None, + recorded_at: Optional[str] = None, + thresholds: Optional[Dict[str, Any]] = None, + passed: Optional[bool] = None, + ) -> Dict[str, Any]: + body = serialize_metric( + key, + value, + inputs, + params, + recorded_at, + thresholds, + passed, + ) + return post_metric( + self._url("log_unit_metric"), + body, + self._headers(), + timeout=self.timeout, + ) + + async def alog_metric(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """Run the synchronous metric transport without blocking the event loop.""" + return await asyncio.to_thread(self.log_metric, *args, **kwargs) + + def _headers(self) -> Dict[str, str]: + headers = { + "X-MODEL-CUID": self.model, + "X-MONITORING": str(self.monitoring), + "X-LIBRARY-VERSION": self.client_version, + } + if self.document: + headers["X-DOCUMENT-TYPE"] = self.document + if self._oidc: + headers["Authorization"] = f"Bearer {self._oidc.token()}" + else: + headers["X-API-KEY"] = self._api_key + headers["X-API-SECRET"] = self._api_secret + return headers + + def _url(self, endpoint: str) -> str: + return urljoin(f"{self.api_host.rstrip('/')}/", endpoint) diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/oidc.py b/packages/validmind-tracking-core/src/validmind_tracking_core/oidc.py new file mode 100644 index 000000000..b3a684556 --- /dev/null +++ b/packages/validmind-tracking-core/src/validmind_tracking_core/oidc.py @@ -0,0 +1,355 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +"""Synchronous OIDC device-flow authentication for tracking clients.""" + +from __future__ import annotations + +import threading +import time +from typing import Any, Callable, Dict, Optional +from urllib.parse import urlparse + +import requests + +from .credentials_store import ( + delete_cached_entry, + expires_at_from_secs, + get_cached_entry, + is_expired, + normalize_audience, + normalize_client_id, + normalize_issuer, + upsert_cached_entry, +) +from .errors import TrackingAuthError + +_OPENID_CONFIG_SUFFIX = "/.well-known/openid-configuration" +_DEFAULT_TIMEOUT = 30.0 +_DEFAULT_SCOPE = "openid profile email offline_access" + + +def _token_entry(payload: Dict[str, Any]) -> Dict[str, Any]: + entry = dict(payload) + if not entry.get("expires_at"): + entry["expires_at"] = expires_at_from_secs(entry.get("expires_in")) + return entry + + +def _bearer_token(entry: Dict[str, Any]) -> str: + issuer = entry.get("issuer", "") + try: + issuer_host = (urlparse(issuer).hostname or "").lower() + except ValueError: + issuer_host = "" + if issuer_host == "login.microsoftonline.com" and entry.get("id_token"): + return entry["id_token"] + token = entry.get("access_token") + if not token: + raise TrackingAuthError("OIDC response did not contain an access token") + return token + + +def _response_json(response: requests.Response) -> Dict[str, Any]: + try: + body = response.json() + except ValueError: + body = {} + return body if isinstance(body, dict) else {} + + +def fetch_openid_configuration( + issuer: str, timeout: float = _DEFAULT_TIMEOUT +) -> Dict[str, Any]: + base = normalize_issuer(issuer) + url = f"{base}{_OPENID_CONFIG_SUFFIX}" + try: + response = requests.get(url, timeout=timeout) + except requests.RequestException as exc: + raise TrackingAuthError( + f"Could not reach OIDC discovery URL {url!r}: {exc}" + ) from exc + if response.status_code != 200: + raise TrackingAuthError( + f"OIDC discovery failed for {url!r}: HTTP {response.status_code} " + f"{response.text[:500]}" + ) + body = _response_json(response) + for key in ("device_authorization_endpoint", "token_endpoint"): + if key not in body: + raise TrackingAuthError( + f"OIDC discovery document from {url!r} is missing {key!r}" + ) + return body + + +def request_device_authorization( + endpoint: str, + client_id: str, + scope: str, + timeout: float = _DEFAULT_TIMEOUT, + audience: Optional[str] = None, +) -> Dict[str, Any]: + payload: Dict[str, str] = {"client_id": client_id, "scope": scope} + normalized_audience = normalize_audience(audience) + if normalized_audience: + payload["audience"] = normalized_audience + try: + response = requests.post( + endpoint, + data=payload, + headers={"Accept": "application/json"}, + timeout=timeout, + ) + except requests.RequestException as exc: + raise TrackingAuthError(f"Device authorization request failed: {exc}") from exc + body = _response_json(response) + if response.status_code != 200: + raise TrackingAuthError( + "Device authorization endpoint rejected the request: " + f"HTTP {response.status_code} {body or response.text[:500]}" + ) + for key in ("device_code", "user_code", "verification_uri"): + if key not in body: + raise TrackingAuthError(f"Device authorization response missing {key!r}") + return body + + +def poll_device_token( + endpoint: str, + client_id: str, + device_code: str, + *, + interval: float = 5.0, + expires_in: float = 900.0, + timeout: float = _DEFAULT_TIMEOUT, + audience: Optional[str] = None, +) -> Dict[str, Any]: + deadline = time.monotonic() + float(expires_in) + current_interval = float(interval) + while time.monotonic() < deadline: + payload: Dict[str, str] = { + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + "device_code": device_code, + "client_id": client_id, + } + normalized_audience = normalize_audience(audience) + if normalized_audience: + payload["audience"] = normalized_audience + try: + response = requests.post( + endpoint, + data=payload, + headers={"Accept": "application/json"}, + timeout=timeout, + ) + except requests.RequestException as exc: + raise TrackingAuthError(f"Token poll request failed: {exc}") from exc + body = _response_json(response) + if response.status_code == 200 and body.get("access_token"): + return _token_entry(body) + error = body.get("error") + if error == "authorization_pending": + time.sleep(current_interval) + continue + if error == "slow_down": + current_interval += 5 + time.sleep(current_interval) + continue + if error == "expired_token": + raise TrackingAuthError("Device login expired before completion") + if error == "access_denied": + raise TrackingAuthError("Device authorization was denied") + raise TrackingAuthError( + f"Token poll failed: HTTP {response.status_code} " + f"error={error!r} {body or response.text[:500]}" + ) + raise TrackingAuthError("Device login timed out waiting for authorization") + + +def refresh_access_token( + endpoint: str, + client_id: str, + refresh_token: str, + scope: Optional[str] = None, + timeout: float = _DEFAULT_TIMEOUT, + audience: Optional[str] = None, +) -> Dict[str, Any]: + payload: Dict[str, str] = { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": client_id, + } + if scope: + payload["scope"] = scope + normalized_audience = normalize_audience(audience) + if normalized_audience: + payload["audience"] = normalized_audience + try: + response = requests.post( + endpoint, + data=payload, + headers={"Accept": "application/json"}, + timeout=timeout, + ) + except requests.RequestException as exc: + raise TrackingAuthError(f"Token refresh request failed: {exc}") from exc + body = _response_json(response) + if response.status_code != 200 or not body.get("access_token"): + raise TrackingAuthError( + f"Token refresh failed: HTTP {response.status_code} " + f"{body or response.text[:500]}" + ) + return _token_entry(body) + + +def run_device_flow( + issuer: str, + client_id: str, + scope: str, + *, + audience: Optional[str] = None, + timeout: float = _DEFAULT_TIMEOUT, + status_callback: Optional[Callable[[Dict[str, Any]], None]] = None, +) -> Dict[str, Any]: + configuration = fetch_openid_configuration(issuer, timeout=timeout) + device = request_device_authorization( + configuration["device_authorization_endpoint"], + client_id, + scope, + timeout=timeout, + audience=audience, + ) + status = { + "verification_uri": device["verification_uri"], + "user_code": device["user_code"], + "verification_uri_complete": device.get("verification_uri_complete"), + } + if status_callback: + status_callback(status) + else: + complete = status.get("verification_uri_complete") or status["verification_uri"] + print( + f"Visit: {complete}\nCode: {status['user_code']}\nWaiting for authorization..." + ) + return poll_device_token( + configuration["token_endpoint"], + client_id, + device["device_code"], + interval=float(device.get("interval", 5)), + expires_in=float(device.get("expires_in", 900)), + timeout=timeout, + audience=audience, + ) + + +class OIDCAuthenticator: + """Load, refresh, or interactively obtain a bearer token.""" + + def __init__( + self, + issuer: str, + client_id: str, + *, + scope: Optional[str] = None, + audience: Optional[str] = None, + timeout: float = _DEFAULT_TIMEOUT, + credentials_path_value=None, + status_callback: Optional[Callable[[Dict[str, Any]], None]] = None, + ): + self.issuer = normalize_issuer(issuer) + self.client_id = normalize_client_id(client_id) + self.scope = scope or _DEFAULT_SCOPE + self.audience = normalize_audience(audience) or None + self.timeout = timeout + self.credentials_path = credentials_path_value + self.status_callback = status_callback + self._entry: Optional[Dict[str, Any]] = None + self._refresh_lock = threading.Lock() + + def initialize(self) -> None: + cached = get_cached_entry( + self.issuer, + self.client_id, + path=self.credentials_path, + audience=self.audience, + ) + if cached and not is_expired(cached): + self._entry = cached + return + if cached and cached.get("refresh_token"): + try: + refreshed = refresh_access_token( + self._token_endpoint(cached), + self.client_id, + cached["refresh_token"], + scope=self.scope, + timeout=self.timeout, + audience=self.audience, + ) + except TrackingAuthError: + delete_cached_entry( + self.issuer, + self.client_id, + path=self.credentials_path, + audience=self.audience, + ) + else: + refreshed.setdefault("refresh_token", cached["refresh_token"]) + self._save(refreshed) + return + entry = run_device_flow( + self.issuer, + self.client_id, + self.scope, + audience=self.audience, + timeout=self.timeout, + status_callback=self.status_callback, + ) + self._save(entry) + + def token(self) -> str: + if self._entry and not is_expired(self._entry): + return _bearer_token(self._entry) + with self._refresh_lock: + if self._entry and not is_expired(self._entry): + return _bearer_token(self._entry) + cached = get_cached_entry( + self.issuer, + self.client_id, + path=self.credentials_path, + audience=self.audience, + ) + if not cached or not cached.get("refresh_token"): + raise TrackingAuthError( + "OIDC access token is missing or expired; initialize the client again" + ) + refreshed = refresh_access_token( + self._token_endpoint(cached), + self.client_id, + cached["refresh_token"], + scope=self.scope, + timeout=self.timeout, + audience=self.audience, + ) + refreshed.setdefault("refresh_token", cached["refresh_token"]) + self._save(refreshed) + return _bearer_token(self._entry) + + def _save(self, entry: Dict[str, Any]) -> None: + saved = _token_entry(entry) + saved["issuer"] = self.issuer + saved["client_id"] = self.client_id + self._entry = saved + upsert_cached_entry( + self.issuer, + self.client_id, + saved, + path=self.credentials_path, + audience=self.audience, + ) + + def _token_endpoint(self, entry: Dict[str, Any]) -> str: + configuration = fetch_openid_configuration(self.issuer, timeout=self.timeout) + return configuration["token_endpoint"] diff --git a/packages/validmind-tracking-core/tests/test_oidc.py b/packages/validmind-tracking-core/tests/test_oidc.py new file mode 100644 index 000000000..601c01042 --- /dev/null +++ b/packages/validmind-tracking-core/tests/test_oidc.py @@ -0,0 +1,91 @@ +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root directory for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial + +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from validmind_tracking_core.credentials_store import ( + get_cached_entry, + upsert_cached_entry, +) +from validmind_tracking_core.oidc import OIDCAuthenticator, _bearer_token + + +class TestOIDCAuthenticator(unittest.TestCase): + def test_bearer_token_matches_entra_hostname(self): + self.assertEqual( + _bearer_token( + { + "issuer": "https://login.microsoftonline.com/tenant/v2.0", + "access_token": "access-token", + "id_token": "id-token", + } + ), + "id-token", + ) + self.assertEqual( + _bearer_token( + { + "issuer": "https://login.microsoftonline.com.evil.example/tenant", + "access_token": "access-token", + "id_token": "id-token", + } + ), + "access-token", + ) + + @patch("validmind_tracking_core.oidc.requests.post") + @patch("validmind_tracking_core.oidc.requests.get") + def test_refreshes_expired_cached_token(self, mock_get, mock_post): + discovery = Mock(status_code=200) + discovery.json.return_value = { + "device_authorization_endpoint": "https://issuer.example/device", + "token_endpoint": "https://issuer.example/token", + } + mock_get.return_value = discovery + refreshed = Mock(status_code=200) + refreshed.json.return_value = { + "access_token": "refreshed-token", + "expires_in": 3600, + } + mock_post.return_value = refreshed + + with TemporaryDirectory() as temp_dir: + credentials_path = Path(temp_dir) / "credentials.json" + upsert_cached_entry( + "https://issuer.example", + "client-1", + { + "access_token": "expired-token", + "refresh_token": "refresh-token", + "expires_at": ( + datetime.now(timezone.utc) - timedelta(hours=1) + ).isoformat(), + }, + path=credentials_path, + ) + auth = OIDCAuthenticator( + "https://issuer.example", + "client-1", + credentials_path_value=credentials_path, + ) + auth.initialize() + + self.assertEqual(auth.token(), "refreshed-token") + self.assertEqual( + mock_post.call_args.args[0], "https://issuer.example/token" + ) + self.assertEqual( + get_cached_entry( + "https://issuer.example", "client-1", path=credentials_path + )["refresh_token"], + "refresh-token", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyproject.toml b/pyproject.toml index 37b9d360c..cd1d8175d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ authors = [ ] dependencies = [ "aiohttp[speedups] (<3.13.1)", + "validmind-tracking-core (>=0.1.0,<0.2.0)", "requests (>=2.28.0,<3.0.0)", "ipywidgets", "kaleido (>=1.2.0,<2.0.0)", @@ -24,7 +25,7 @@ dependencies = [ "numpy (>=2.3,<3.0.0) ; python_version >= '3.14'", "openai (>=1)", "pandas (>=2.0.3,<3.0.0)", - "plotly (>=6.0.0)", + "plotly (>=6.0.0,<7.0.0)", "polars", "python-dotenv", "scikit-learn", @@ -195,3 +196,12 @@ profile = "black" [tool.uv] exclude-newer = "7 days" + +[tool.uv.sources] +validmind-tracking-core = { workspace = true } + +[tool.uv.workspace] +members = [ + "packages/validmind-tracking-core", + "packages/validmind-metrics", +] diff --git a/tests/test_api_client.py b/tests/test_api_client.py index 57e506f92..09461d230 100644 --- a/tests/test_api_client.py +++ b/tests/test_api_client.py @@ -148,6 +148,24 @@ def test_get_api_model(self): model = api_client.get_api_model() self.assertEqual(model, "your_model") + @patch("requests.post") + def test_log_metric_is_safe_inside_running_event_loop(self, mock_post): + mock_post.return_value = MockResponse(200, json={"ok": True}) + + async def handler(): + first = api_client.log_metric("accuracy", 0.95) + second = api_client.log_metric("accuracy", 0.96) + return first, second + + self.assertEqual(asyncio.run(handler()), ({"ok": True}, {"ok": True})) + self.assertEqual(mock_post.call_count, 2) + self.assertTrue( + all( + call.args[0].endswith("/log_unit_metric") + for call in mock_post.call_args_list + ) + ) + @patch("requests.get") def test_init_missing_model_id(self, mock_requests_get): mock_requests_get.return_value = Mock() diff --git a/uv.lock b/uv.lock index 882938164..128a60f41 100644 --- a/uv.lock +++ b/uv.lock @@ -18,6 +18,13 @@ resolution-markers = [ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P7D" +[manifest] +members = [ + "validmind", + "validmind-metrics", + "validmind-tracking-core", +] + [[package]] name = "aiodns" version = "3.6.1" @@ -11298,6 +11305,7 @@ dependencies = [ { name = "tabulate" }, { name = "tiktoken" }, { name = "tqdm" }, + { name = "validmind-tracking-core" }, ] [package.optional-dependencies] @@ -11494,7 +11502,7 @@ requires-dist = [ { name = "numpy", marker = "python_full_version >= '3.14'", specifier = ">=2.3,<3.0.0" }, { name = "openai", specifier = ">=1" }, { name = "pandas", specifier = ">=2.0.3,<3.0.0" }, - { name = "plotly", specifier = ">=6.0.0" }, + { name = "plotly", specifier = ">=6.0.0,<7.0.0" }, { name = "polars" }, { name = "presidio-analyzer", marker = "python_full_version >= '3.14' and extra == 'pii-detection'", specifier = "<2.2.360" }, { name = "presidio-analyzer", marker = "python_full_version < '3.14' and extra == 'pii-detection'" }, @@ -11549,6 +11557,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'huggingface'", specifier = ">=4.32.0,<5.0.0" }, { name = "transformers", marker = "extra == 'llm'", specifier = ">=4.32.0,<5.0.0" }, { name = "transformers", marker = "extra == 'nlp'", specifier = ">=4.32.0,<5.0.0" }, + { name = "validmind-tracking-core", editable = "packages/validmind-tracking-core" }, { name = "xgboost", marker = "extra == 'all'", specifier = ">=1.5.2,<3.1" }, { name = "xgboost", marker = "extra == 'xgboost'", specifier = ">=1.5.2,<3.1" }, ] @@ -11575,6 +11584,28 @@ dev = [ { name = "twine", specifier = ">=4.0.2,<5" }, ] +[[package]] +name = "validmind-metrics" +version = "0.1.0" +source = { editable = "packages/validmind-metrics" } +dependencies = [ + { name = "validmind-tracking-core" }, +] + +[package.metadata] +requires-dist = [{ name = "validmind-tracking-core", editable = "packages/validmind-tracking-core" }] + +[[package]] +name = "validmind-tracking-core" +version = "0.1.0" +source = { editable = "packages/validmind-tracking-core" } +dependencies = [ + { name = "requests" }, +] + +[package.metadata] +requires-dist = [{ name = "requests", specifier = ">=2.28.0,<3.0.0" }] + [[package]] name = "virtualenv" version = "21.3.1" diff --git a/validmind/api_client.py b/validmind/api_client.py index 69d6e43a8..1d4491161 100644 --- a/validmind/api_client.py +++ b/validmind/api_client.py @@ -19,6 +19,8 @@ import aiohttp import requests from aiohttp import FormData +from validmind_tracking_core.errors import TrackingAPIError +from validmind_tracking_core.metrics import post_metric, serialize_metric from .__version__ import __version__ from .client_config import client_config @@ -995,7 +997,7 @@ def log_text( return _render_logged_text(logged_text) -async def alog_metric( +def _send_metric_sync( key: str, value: Union[int, float], inputs: Optional[List[str]] = None, @@ -1004,38 +1006,48 @@ async def alog_metric( thresholds: Optional[Dict[str, Any]] = None, passed: Optional[bool] = None, ): - """See log_metric for details.""" - if not key or not isinstance(key, str): - raise ValueError("`key` must be a non-empty string") - - if value is None: - raise ValueError("Must provide a value for the metric") - - # Validate that value is a scalar (int or float) - if not isinstance(value, (int, float)): - raise ValueError( - "Only scalar values (int or float) are allowed for logging metrics." + """Send one metric without creating or depending on an event loop.""" + _ensure_fresh_oidc_token() + try: + return post_metric( + _get_url("log_unit_metric"), + serialize_metric( + key, + value, + inputs, + params, + recorded_at, + thresholds, + passed, + encoder=NumpyEncoder, + ), + _get_api_headers(), + timeout=float(os.getenv("VM_API_TIMEOUT", 30)), ) + except TrackingAPIError as e: + _raise_for_api_error(e.status_code, e.response_text) - if thresholds is not None and not isinstance(thresholds, dict): - raise ValueError("`thresholds` must be a dictionary or None") +async def alog_metric( + key: str, + value: Union[int, float], + inputs: Optional[List[str]] = None, + params: Optional[Dict[str, Any]] = None, + recorded_at: Optional[str] = None, + thresholds: Optional[Dict[str, Any]] = None, + passed: Optional[bool] = None, +): + """See log_metric for details, without blocking the current event loop.""" try: - return await _post( - "log_unit_metric", - data=json.dumps( - { - "key": key, - "value": value, - "inputs": inputs or [], - "params": params or {}, - "recorded_at": recorded_at, - "thresholds": thresholds or {}, - "passed": passed if passed is not None else None, - }, - cls=NumpyEncoder, - allow_nan=False, - ), + return await asyncio.to_thread( + _send_metric_sync, + key, + value, + inputs, + params, + recorded_at, + thresholds, + passed, ) except Exception as e: logger.error("Error logging metric to ValidMind API") @@ -1068,16 +1080,19 @@ def log_metric( thresholds (Dict[str, Any], optional): Thresholds for the metric passed (bool, optional): Whether the metric passed validation thresholds """ - return run_async( - alog_metric, - key=key, - value=value, - inputs=inputs, - params=params, - recorded_at=recorded_at, - thresholds=thresholds, - passed=passed, - ) + try: + return _send_metric_sync( + key, + value, + inputs, + params, + recorded_at, + thresholds, + passed, + ) + except Exception as e: + logger.error("Error logging metric to ValidMind API") + raise e def generate_test_result_description(test_result_data: Dict[str, Any]) -> str: