diff --git a/anomstack/observability.py b/anomstack/observability.py new file mode 100644 index 0000000..0d3c2d1 --- /dev/null +++ b/anomstack/observability.py @@ -0,0 +1,100 @@ +"""Opt-in PostHog error tracking and OpenTelemetry application logs.""" + +import atexit +import logging +import os +import time + +_client = None + + +def configure_observability(): + """Initialize once per process; leave non-demo installations unchanged.""" + global _client + if _client is not None: + return _client + key = os.getenv("POSTHOG_API_KEY") + if not key or os.getenv("POSTHOG_OBSERVABILITY_ENABLED", "false").lower() != "true": + return None + + from posthog import Posthog + + host = os.getenv("POSTHOG_HOST", "https://us.i.posthog.com").rstrip("/") + _client = Posthog( + key, + host=host, + enable_exception_autocapture=True, + super_properties={"app": "anomstack", "environment": "demo"}, + ) + atexit.register(_client.shutdown) + + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.resources import Resource + + provider = LoggerProvider( + resource=Resource.create( + { + "service.name": "anomstack-dashboard", + "deployment.environment.name": "demo", + } + ) + ) + provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter( + endpoint=f"{host}/i/v1/logs", + headers={"Authorization": f"Bearer {key}"}, + timeout=5, + ) + ) + ) + handler = LoggingHandler(level=logging.INFO, logger_provider=provider) + # Export application logs only, avoiding exporter recursion and SDK payload logs. + for name in ("anomstack", "anomstack_dashboard"): + logger = logging.getLogger(name) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + atexit.register(provider.shutdown) + logging.getLogger("anomstack").info("PostHog observability initialized") + return _client + + +class ObservabilityMiddleware: + """Capture request failures without changing responses or recording request bodies.""" + + def __init__(self, app, client): + self.app = app + self.client = client + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self.app(scope, receive, send) + started = time.monotonic() + status = 500 + + async def track_send(message): + nonlocal status + if message["type"] == "http.response.start": + status = message["status"] + await send(message) + + try: + await self.app(scope, receive, track_send) + except Exception as exc: + self.client.capture_exception(exc, distinct_id="anomstack-demo-server") + raise + finally: + if scope.get("path") != "/health": + route = scope.get("route") + logging.getLogger("anomstack.http").log( + logging.ERROR if status >= 500 else logging.INFO, + "HTTP request completed", + extra={ + "http.request.method": scope["method"], + "http.route": getattr(route, "path", "unmatched"), + "http.response.status_code": status, + "duration_ms": round((time.monotonic() - started) * 1000, 2), + }, + ) diff --git a/constraints.txt b/constraints.txt index b2ce369..de06524 100644 --- a/constraints.txt +++ b/constraints.txt @@ -289,6 +289,7 @@ googleapis-common-protos==1.75.3 # via # google-api-core # grpcio-status + # opentelemetry-exporter-otlp-proto-http gql==4.0.0 # via dagster-graphql graphene==3.4.3 @@ -604,7 +605,28 @@ oauthlib==3.3.1 openai==3.8.0 # via langchain-openai opentelemetry-api==1.44.0 - # via google-api-core + # via + # google-api-core + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp-proto-common==1.44.0 + # via opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-http==1.44.0 + # via + # -r requirements.txt + # anomstack +opentelemetry-proto==1.44.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.44.0 + # via + # -r requirements.txt + # anomstack + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.65b0 + # via opentelemetry-sdk orjson==3.12.0 # via # langgraph-sdk @@ -714,6 +736,7 @@ protobuf==6.33.6 # googleapis-common-protos # grpcio-health-checking # grpcio-status + # opentelemetry-proto # proto-plus # streamlit psutil==7.2.2 @@ -892,6 +915,7 @@ requests==2.34.2 # gql # jupyterlab-server # langsmith + # opentelemetry-exporter-otlp-proto-http # posthog # requests-oauthlib # requests-toolbelt @@ -1068,6 +1092,9 @@ typing-extensions==4.16.0 # langsmith # openai # opentelemetry-api + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-sdk + # opentelemetry-semantic-conventions # posthog # pydantic # pydantic-core diff --git a/dashboard/README.md b/dashboard/README.md index 4dde4a3..627cb01 100644 --- a/dashboard/README.md +++ b/dashboard/README.md @@ -28,3 +28,32 @@ To stop: ```bash make kill-dashboardd ``` + +### Demo observability + +The Fly demo sends browser analytics and server telemetry to Deepwell's PostHog +`dev` project (148051). `profiles/demo.env` holds the deployment configuration. + +- `POSTHOG_FRONTEND_API_KEY`: browser analytics, exceptions, and session replay. +- `POSTHOG_API_KEY` / `POSTHOG_HOST`: existing LLM telemetry and server errors/logs. +- `POSTHOG_OBSERVABILITY_ENABLED=true`: opt in to dashboard request logs and server + exception capture. Other installations remain opted out by default. + +Application logs use OpenTelemetry HTTP export to `/i/v1/logs`, with service name +`anomstack-dashboard`. HTTP logs include method, route template, status, and duration; +request bodies, query strings, and headers are not included. This covers dashboard +application logs, not nginx, Dagster daemon, or machine-wide stdout. + +Replay uses the project's recording settings and masks input values. The existing +Replay Vision scanners process eligible recordings and emit Signals. Those scanners +are shared with other apps using the dev project. Filter analytics/replays to +`anomstack-live-demo.fly.dev` when inspecting this demo. + +Useful project links: +- https://us.posthog.com/project/148051/error_tracking +- https://us.posthog.com/project/148051/replay +- https://us.posthog.com/project/148051/replay-vision +- https://us.posthog.com/project/148051/logs + +Setup references: https://posthog.com/docs/logs/installation/python and +https://posthog.com/docs/error-tracking/installation/python. diff --git a/dashboard/app.py b/dashboard/app.py index 8658aab..98060a4 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -88,6 +88,12 @@ def get_posthog_script(): log=log, ) +from anomstack.observability import ObservabilityMiddleware, configure_observability + +telemetry = configure_observability() +if telemetry is not None: + app.add_middleware(ObservabilityMiddleware, client=telemetry) + # Set the app state app.state = AppState() diff --git a/dashboard/constants.py b/dashboard/constants.py index 5be1b64..c6d66e4 100644 --- a/dashboard/constants.py +++ b/dashboard/constants.py @@ -14,6 +14,12 @@ !function(t,e){var o,n,p,r;e.__SV||(window.posthog=e,e._i=[],e.init=function(i,s,a){function g(t,e){var o=e.split(".");2==o.length&&(t=t[o[0]],e=o[1]),t[e]=function(){t.push([e].concat(Array.prototype.slice.call(arguments,0)))}}(p=t.createElement("script")).type="text/javascript",p.crossOrigin="anonymous",p.async=!0,p.src=s.api_host.replace(".i.posthog.com","-assets.i.posthog.com")+"/static/array.js",(r=t.getElementsByTagName("script")[0]).parentNode.insertBefore(p,r);var u=e;for(void 0!==a?u=e[a]=[]:a="posthog",u.people=u.people||[],u.toString=function(t){var e="posthog";return"posthog"!==a&&(e+="."+a),t||(e+=" (stub)"),e},u.people.toString=function(){return u.toString(1)+".people (stub)"},o="init capture register register_once register_for_session unregister unregister_for_session getFeatureFlag getFeatureFlagPayload isFeatureEnabled reloadFeatureFlags updateEarlyAccessFeatureEnrollment getEarlyAccessFeatures on onFeatureFlags onSurveysLoaded onSessionId getSurveys getActiveMatchingSurveys renderSurvey canRenderSurvey canRenderSurveyAsync identify setPersonProperties group resetGroups setPersonPropertiesForFlags resetPersonPropertiesForFlags setGroupPropertiesForFlags resetGroupPropertiesForFlags reset get_distinct_id getGroups get_session_id get_session_replay_url alias set_config startSessionRecording stopSessionRecording sessionRecordingStarted captureException loadToolbar get_property getSessionProperty createPersonProfile opt_in_capturing opt_out_capturing has_opted_in_capturing has_opted_out_capturing clear_opt_in_out_capturing debug getPageViewId captureTraceFeedback captureTraceMetric".split(" "),n=0;n