From 5d5d8c1606e1e8faaa725fc2e205f7087c80d847 Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 12:55:32 +0200 Subject: [PATCH 01/69] docs: plan API monitoring (M16): request rollups, every API stack, dashboards, service health and status pages Records the decisions for #63 and its steps #64-#71 in the architecture and the roadmap: an API is an analytics property of kind api, no row per request or address is stored, percentiles come from mergeable sketches, uptime checks and health pulls go through the webhook dialer's address checks, and status pages on custom domains are verified by CNAME and TXT. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 344 ++++++++++++++++++++++++++++++++++++++++++- docs/roadmap.md | 50 +++++++ 2 files changed, 392 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index fb8f4b59..efed59fb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,8 +79,9 @@ panel (embedded SPA) ─────────► /v1 + WS ────── - Go, `net/http`, OpenAPI 3.1 contract in `openapi/openapi.yaml`, strict server generated with oapi-codegen. Postgres through pgx and sqlc; goose migrations. - Background work on River (Postgres-backed queue): outbound e-mail, webhooks, notification - fan-out, attachment cleanup, retention, and migrations on their own `migrations` queue (see - "Moving to Yuva"). + fan-out, attachment cleanup, retention, migrations on their own `migrations` queue (see + "Moving to Yuva"), and uptime checks, health pulls and certificate orders on their own `monitoring` + queue (see "API monitoring"). - Realtime: WebSocket (`coder/websocket`). Every change that members see live writes a row to `events` in the same transaction, last, under a per-workspace advisory lock, so event ids within a workspace follow commit order; the transaction's `NOTIFY` carries the id. One listener per @@ -118,6 +119,10 @@ panel (embedded SPA) ─────────► /v1 + WS ────── | `/client/v1` | widget, mobile SDKs | channel public key + contact session, or a reader token on the discussion endpoints | | `/client/v1/analytics/{key}` | the analytics script, `useyuva/analytics`, the mobile analytics modules | an analytics property's public key; browsers only from the property's domains | | `/v1/analytics/shared/{token}` | read-only analytics shares | a share token (and its password session) | +| `/otlp/{propertyId}/v1/traces` | OpenTelemetry exporters of host APIs | workspace API key with `analytics:write` (Bearer) | +| `/prometheus/api/v1/*` | Grafana and other Prometheus clients | workspace API key with `analytics:read` (Bearer) | +| `/status/{slug}/`, verified status page hosts | visitors of public status pages | none; a page password or a member session for private pages | +| `/.well-known/acme-challenge/{token}` | the ACME certificate authority (`YUVA_STATUS_TLS=acme`) | none; answers only tokens of pending orders | | `/ingress/email` | edge Worker, MTAs | HMAC signature over the raw message | | `/ingress/ses` | SES bounce and complaint notifications via SNS | SNS signature | | `/s/{token}`, `/m/{token}`, `/c/{token}`, `/i/{token}` | people who receive list and broadcast mail | a token sealed under the master key; an image token | @@ -1696,6 +1701,320 @@ in six steps (M15.1–M15.6 in the roadmap). Decisions: no record id of any workspace leaves the browser. A load balancer's sticky cookie must skip the script and `/client/v1/analytics/`, or the promise of nothing on the device breaks. +### API monitoring (M16) + +A workspace watches its own APIs and services, written in any language: request volume, errors and +latency per route and consumer, uptime checks, the health of each service and its dependencies, +alerts, dashboards and public status pages. Plan in the M16 umbrella issue #63; it ships in eight +steps (M16.1–M16.8 in the roadmap). It counts requests, never visitors, and keeps no row per +request. Decisions: + +- **An API is an analytics property.** `analytics_properties.kind` is `site` (every M15 property) or + `api`, fixed at creation. An API property keeps the property's inbox link, access rules, time zone, + `retention_months`, public key (for app-side records only), key scopes (`analytics:read`, `write`, + `manage`), annotations, shares, exports, report subscriptions and the one query operation, so every + M15 table that hangs off a property works for an API without a parallel table. Its `domains` (0–50) + are the API's hostnames, its `platforms` the apps allowed to send app-side records (`ios`, + `android`, or none); it takes no goals, props, funnels or consent features, and `audience` does not + apply. New columns: `apdex_ms` (500) and `monitor_hour_days` (90). The collection endpoint and the + site views refuse an API property; the API endpoints refuse a site. +- **A record.** Method (the RFC 9110 methods and `PATCH`, else `_OTHER`), route, status (100–599, or 0 + when no response was sent), duration (microseconds), request and response size, time, and optionally + `consumer` (1–100 characters, chosen by the host: an API key's name or a client id, never a person's + address; e-mail-shaped values become `(scrubbed)`), `api_version` (≤ 40), `environment` (≤ 40, + `production` by default), `service` (≤ 100), `release` (≤ 80, only for deploy markers), `ip` (only + to find the country) and `sample_rate` (1–10,000, 1 by default). No body, header, query string, + fragment, user agent or address is accepted or kept. 5xx and 0 are errors, 4xx client errors; both + rates are shown. +- **Routes, never raw paths.** A record names its route pattern (`/users/{id}`: `r.Pattern` from + `http.ServeMux`, the framework's router, OpenTelemetry `http.route`). When it can only give `path`, + ingest infers a pattern: segments that are numbers, UUIDs, dates, hex or base64url of 16 or more + characters, e-mail-shaped or longer than 40 characters become `{id}`, and `?`, `#` and what follows + are cut first. A record without a pattern that answered 404 or 405 goes to `(unmatched)`, so scanners + never create routes. `monitor_routes` marks inferred patterns; members hide a route or map inferred + ones onto a pattern of their own (route rules, applied at ingest from then on). Caps per property: + 1,000 routes, 10,000 consumers, 50 API versions, 200 app versions, 20 environments; 100 services per + workspace. A value past its cap is counted as `(other)` and the panel says which cap was reached. +- **Percentiles from mergeable sketches.** Durations go into a sparse log-bucketed histogram with base + 2^(1/32) (the bucket layout of an OpenTelemetry exponential histogram at scale 5), so a reported + percentile is within 1.1% of the exact value; sizes into one at scale 2 (within 9%). A sketch is a + `bytea` of varint (bucket index delta, count) pairs with its scale, stored next to exact `requests`, + `errors`, `client_errors`, sum, minimum and maximum columns. Merging adds counts per bucket, so it is + exact and associative: a day's p99 folded from 1,440 minutes is the p99 of that day's records to the + bucket width, which t-digest-style sketches do not promise. Sketches are merged in Go (ingest, folds, + queries); SQL sums the counters and never opens a sketch. No Postgres extension. Apdex (satisfied at + most T, tolerating at most 4T, T the route's or the property's `apdex_ms`) and the latency heatmap + read the same buckets. +- **Storage: no row per request.** Each process aggregates records in memory into the open minute of + their core key and the open hour of each extra dimension. Every 10 seconds it upserts its cumulative + totals under its own random `writer` id, replacing its previous upsert rather than adding to it, so + processes never contend for a row and a repeated flush changes nothing; a bucket is flushed a last + time 2 minutes after it ends and leaves memory. Records for a bucket already gone (at most 72 hours + old, as for M15) are written under a fresh writer and mark their hour and day dirty. Shutdown flushes + everything, as the analytics flusher drains. The core key is (environment, service, route, method, + status, API version, source, platform); the extra dimensions are consumer, country and app version, + each kept with route, method and status class. Tables, all with `workspace_id`: + - `monitor_minute`: core rows by UTC minute, partitioned by day and kept `YUVA_MONITOR_MINUTE_DAYS` + (8: today and a 7-day alert baseline); partitions are made by the aggregator and made and dropped + by the `analytics_partitions` job under the same advisory lock as M15's raw partitions. + - `monitor_hour`, `monitor_dim_hour`: by the property's local hour, partitioned by month, kept the + property's `monitor_hour_days` (7–400). + - `monitor_day`, `monitor_dim_day`: by local day, kept `retention_months`. + Hours are local, so 24 of them (23 or 25 at a DST change) make the property's day in every time zone, + half-hour offsets included; a time zone change applies from the next local day. The `monitor_rollup` + job on the `analytics` queue folds a closed hour's minute rows into one row per key, compacts its + dimension rows to one writer, and folds finished local days, tracked in `monitor_rollup_hours` + (built, dirty) like `analytics_rollup_days`. Queries read folded rows for closed hours and minute and + writer rows for open ones. An hour holds at most 50,000 dimension rows per property; later new + combinations of that hour count as `(other)` and the hour is marked capped. Dictionaries give the + rollups integer ids: `monitor_routes` (pattern, inferred, hidden, `apdex_ms`, first and last seen) + and `monitor_names` (consumers, API versions, app versions, environments, countries per property). +- **Sampling.** Off by default: records are aggregated on arrival, so cost grows with distinct keys, not + with requests. An SDK may sample successful requests (`sampleRate`), never 4xx, 5xx or failed ones; + a record carries its weight, counters and buckets add it, and `sampled` keeps the weighted requests + of each row, so views, exports and the Prometheus endpoint mark a range that holds any as estimated. + An OTLP span weighs the adjusted count of its tracestate `ot=th` threshold (OpenTelemetry's + probability sampling, still a Development spec) and 1 without one, so a head-sampled exporter + undercounts; the guide says to export every server span to Yuva. +- **Ingest.** + - `POST /v1/analytics/properties/{id}/requests` (`analytics:write`): 1–1,000 records, gzip + accepted, `202` with `accepted` and per-record `errors`, like server events; only a body or + authentication problem refuses the request. + - `POST /otlp/{propertyId}/v1/traces`: OTLP/HTTP, so an exporter needs only + `OTEL_EXPORTER_OTLP_ENDPOINT=/otlp/` and + `OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer `. Binary protobuf and + JSON by `Content-Type`, gzip, as the OTLP specification requires; the few fields used are read with + `protowire` (already in the module graph), so no OTLP module is added. Only `SERVER` spans count: + `http.request.method`, `http.route`, `http.response.status_code`, end minus start, `url.path` for + inference only, `client.address` for the country, `http.request.body.size` and + `http.response.body.size` when present; resource `service.name`, `deployment.environment.name` + (or the older `deployment.environment`), `service.version` as the release and `service.instance.id`; + span attributes `yuva.consumer` and `yuva.api_version`. Other spans are ignored. The answer is `200` + with an `ExportTraceServiceResponse` whose `partial_success` counts invalid spans, or `429`/`503` + with `Retry-After`. + - App-side: `POST /client/v1/analytics/{key}/requests` from the Swift and Kotlin modules, a batch of + up to 100 records with `a.platform` in the property's `platforms` and `a.app_version`; source + `app`, no consumer, sent only under the property's `collection` rules exactly like app events. + - On every path the address is shortened and looked up for the country only (no region or city) in + the first function that sees it, then dropped. Limits per property + (`YUVA_MONITOR_RECORDS_PER_MINUTE`, 600,000) and per key; in-memory keys per process are capped + (`YUVA_MONITOR_BUFFER_KEYS`, 200,000; full answers `503`). `monitor_ingest_stats` counts accepted, + invalid, rate, over_limit and buffer per property and day. +- **SDKs (M16.2).** `sdk/go/analytics` gets `Middleware` for `net/http`: it wraps the mux and reads + `r.Pattern` after the handler ran (`ServeMux` sets it on the same request), without its method and + host, or a `Route func(*http.Request) string`; batches of 500 or every 5 seconds through the existing + queue and backoff. `useyuva/server` gets `createMonitor` with middleware for srvx, h3, Nitro and Nuxt, + Hono, Express, Fastify and Next.js route handlers, each taking the pattern from its router and flushing + through `waitUntil` or the server's close hook. Swift `YuvaAnalytics` and Kotlin `yuva-analytics` + time the host's calls to its own API (`URLSession` and OkHttp interceptors) with routes from a + host-supplied mapping, off until the host turns it on; the privacy manifest adds + `NSPrivacyCollectedDataTypePerformanceData` (not linked, no tracking, purposes analytics and app + functionality) and the data safety guide adds "App info and performance › Diagnostics". +- **Analysis (M16.3).** The query operation takes `dataset: api` with metrics `requests`, + `requests_per_minute`, `errors`, `error_rate`, `client_errors`, `client_error_rate`, `p50`, `p75`, + `p90`, `p95`, `p99`, `avg_duration`, `max_duration`, `apdex`, `request_bytes`, `response_bytes`, + `response_size_p50`, `response_size_p95` and `conversations`; dimensions route, method, status, + status class, API version, environment, service, source, platform, consumer, country, app version and + `time:minute` (inside the minute tier, at most 1,440 buckets), `time:hour`, `time:day`, `time:week`, + `time:month`. `range` also takes `15m`, `1h`, `6h` and `24h` and minute-aligned `since` and `until`. + Any combination of core dimensions filters and breaks down from every tier; at most one extra + dimension (consumer, country or app version) joins them, from hours and days (`422 + dimension_combination` otherwise). Comparisons, the 4 query slots per workspace and the cache are + M15's. New routes are first seen in the range, silent routes were called in the previous period and + not since. A new `release` per service and environment writes a `deploy` annotation, which gains a + time (`at`) so minute charts place it; Yuva's own annotations stay read-only. +- **Uptime checks (M16.4).** `monitor_checks`: name, `http` or `https` URL without credentials, `GET` + or `HEAD`, up to 5 headers (values sealed under the master key), expected statuses (200–399), an + optional keyword the first 64 KiB must or must not contain, timeout (1–30 s, 10), interval (30 s to + 24 h; at least `YUVA_MONITOR_MIN_INTERVAL`, 60 s with `YUVA_SIGNUP=open`, else 30 s), failures in a + row before down (2), certificate warning days (14), up to 5 redirects, an optional service and + property. The `monitor_checks_due` job (every 10 s) takes due checks with `FOR UPDATE SKIP LOCKED`, + moves `next_run_at` and queues one `monitor_check` job per run, unique by check and slot, on the + `monitoring` queue (`YUVA_MONITOR_WORKERS`, 8), so slow endpoints never delay rollups or mail. Requests + go through the webhook client's dialer: resolved once, refused on loopback, private, link-local, + CGNAT, multicast and reserved addresses, connected to the resolved address only, every redirect hop + checked again; `User-Agent: Yuva-Uptime/`, no cookies, no body sent, at most 64 KiB read; + DNS, connect, TLS, first-byte and total times recorded. `YUVA_MONITOR_ALLOW_PRIVATE=true` lets checks + and health pulls reach private addresses (internal services of a self-hosted install); link-local and + cloud metadata addresses are refused even then. A failed run is tried again after 20 s before it + counts. When at least 20 checks run in a minute and more than half of them fail, that minute's + failures are recorded `unknown`, since Yuva's own network is the likelier fault. Checks are their own + `uptime` dataset (`up`, `uptime`, `response_time` percentiles, `ttfb`, certificate days left), not a + source of API records, because a probe is not a consumer and would skew a route's numbers. Results go + to `monitor_check_results` (partitioned by day, `YUVA_MONITOR_CHECK_DAYS`, 30) and fold into + `monitor_check_days` (runs, failures, down seconds, a response-time sketch; 400 days). Caps per + workspace: `YUVA_MONITOR_CHECKS` (50 with open sign-up, else 500), and at most 10 checks on one host. +- **Alerts and reports (M16.4).** `monitor_alert_rules` per property, route, service or check: error + rate, client error rate, p95, p99 or Apdex past a threshold over a window (5–60 minutes, with at least + `min_requests`, 20); traffic spike or drop (the last 15 closed minutes at 3× or at most half of the + same 15 minutes averaged over the 7 days before, with a baseline of at least 50 requests); a route + going silent (none for 15 minutes after averaging one a minute over the previous day); a check or + service changing state; a certificate inside its warning days. The `monitor_alerts` job runs every + minute over closed minutes. An alert is one row per (rule, key) while open: it fires once when the + condition holds and resolves once when the value has been back inside the threshold with a 10% margin + for a whole window. Both write a `monitor.alert` stored event and webhook (`state` + `firing` or `resolved`) and an annotation, and reach the rule's member recipients by e-mail and Web + Push (urgency `high`, the alert as `Topic`, a one-hour TTL); M15's one-shot `analytics_alerts` stay as + they are, because these open and close. A new API property gets one rule: 5xx rate at 5% over 5 + minutes, sent to its creator. Weekly and monthly reports use `analytics_subscriptions` and the + `analytics_reports` job with an API template (requests, error rates, p95, slowest and most failing + routes, uptime, incidents). Support in aggregate: a property linked to an inbox gets `conversations` + per local day of the inbox next to its errors, by time only; no request is ever joined to a person or + a conversation. +- **Sharing, export and retention (M16.5).** Share links of an API property list the views + `api_overview`, `api_routes` and `api_consumers` (consumer names only when listed) with M15's tokens, + passwords, expiry, embedding and limits. Exports `view` and `daily` take `dataset: api` and `uptime` + in CSV or NDJSON; there is no `raw` export because there are no raw rows. Retention: minutes 8 days + (server-wide, partitions dropped), hours `monitor_hour_days`, days and dimension days + `retention_months`, check results 30 days, check days, state changes and service history 400 days; + `DELETE …/consumers/{id}` erases a consumer's rows and name. Deleting a property, a service or a + check deletes its rows; workspace `retention_days` does not apply. +- **Dashboards (M16.6).** `monitor_dashboards` belong to the workspace: name, a JSON document + (`MonitorDashboardDocument`, `schema: 1`: a default time range, auto-refresh off or 30 s to 15 min, + up to 10 variables of kind property, route, consumer, environment, service or custom, and up to 50 + panels on a 24-column grid, each with up to 5 queries, a type (time series, bar, table, stat, gauge, + latency heatmap, health, text), unit, decimals, legend and thresholds) and a `version` for concurrent + edits, as `email_designs` do. A panel query is the query operation's own body (any dataset: traffic, + support, api, uptime, health) with a property or `$property`, and `$variable` values substituted into + filters, so a panel and the fixed views can never disagree. Text panels are Markdown rendered by + goldmark and cleaned by bluemonday. A threshold on an `api`, `uptime` or `health` metric becomes an + alert rule in one step. Dashboards are in `/v1`, imported and exported as the document; starter + dashboards for an API, a website and an app are documents shipped in the binary. Dashboard share + links reuse `analytics_shares` with `dashboard_id` in place of `property_id` (exactly one is set); + such a link fixes the variable values it may show and reads only the properties its panels named + when it was made. +- **Prometheus-compatible endpoint (M16.6).** `/prometheus/api/v1/` `query`, `query_range`, `labels`, + `label/{name}/values`, `series` and `status/buildinfo`, GET or form POST, Prometheus's JSON answers, + for a key with `analytics:read` (Grafana sends it as an `Authorization` header). Series are gauges per + step: `yuva_api_requests`, `yuva_api_requests_per_second`, `yuva_api_errors`, `yuva_api_error_ratio`, + `yuva_api_duration_seconds{quantile="…"}` (any quantile), `yuva_api_apdex`, `yuva_uptime_up`, + `yuva_uptime_response_seconds{quantile="…"}`, `yuva_service_state` (0 healthy, 1 degraded, 2 down), + `yuva_web_visitors`, `yuva_web_pageviews`; labels are the query dimensions plus `property`. The PromQL + subset is selectors with `=`, `!=`, `=~` and `!~`; `sum`, `avg`, `min`, `max` and `count` `by` or + `without`; `topk` and `bottomk`; arithmetic with number literals and between two vectors with the same + labels; number literals alone (so data-source tests pass). It compiles to one query, so aggregating a + duration series merges its sketches (`max by (route)` of the p95 is the route's p95, not the largest + of several) and the label rules of the query layer apply. No range vectors, `rate`, `increase`, + `histogram_quantile`, offsets or subqueries: rollups are per-step amounts, not counters; anything else + answers `400` `bad_data`. Steps are at least 60 s and are rounded up to the tier the range allows; at + most 11,000 points per series. +- **Service health (M16.7).** `monitor_services` (name and environment unique in the workspace, + optional API property, `push` or `pull` with a health URL and interval of 30 s to 10 min, a + silence timeout of 1 minute to 1 day, 3 intervals by default, version, state since) with + `monitor_dependencies` (name, kind `database`, `cache`, `queue`, `http` or `other`, critical or not, + last latency; 50 per service) and `monitor_instances` (instance id, version, last report; 200 per + service, gone after 24 hours of silence). A service is made by its first health report, by a + pull service a member adds, or by the first record or span that names it. Reports are pushed to + `POST /v1/monitoring/health` (`analytics:write`: service, environment, version, instance, status, + checks with name, kind, status, latency, critical and a message of at most 200 characters shown to + members only, never on a status page) or pulled through the uptime runner and its address checks; a + pulled endpoint answers `application/health+json` (the IETF draft health check format), Spring Boot + Actuator's `status` and `components`, or anything else, judged by its status code. State per + dependency and service is `healthy`, `degraded` or `down` (`unknown` before a first report): a + critical dependency down makes the service down, a non-critical one degraded; no report within the + silence timeout makes it down; a linked check that is down makes it down; the linked property's 5xx + rate (5% by default, at least 20 requests) or p95 (a threshold per service) over 5 minutes makes it + degraded. Recovery needs two healthy evaluations in a row. The `monitor_states` job (every 30 s) and + every report evaluate; each change writes `monitor_state_changes` (subject, state, reason, time; 400 + days), from which history and uptime per day are computed. `monitor_health_hours` keeps reports and + a latency sketch per dependency and UTC hour for 90 days. A service turning degraded or down opens one + automatic incident (one open per service), and 5 minutes healthy resolves it; state-change rules of + M16.4 alert on both. `sdk/go/health` and `useyuva/server` `createHealth` register checks with + timeouts, serve `application/health+json` and push reports; OTLP resource attributes name the + service for OTLP senders. +- **Incidents.** `monitor_incidents` (origin `auto` or `manual`, kind `incident` or `maintenance`, + title, state `investigating`, `identified`, `monitoring` or `resolved`, impact, affected services, + dependencies and checks, start, end, scheduled window, `published`) with `monitor_incident_updates` + (state, text of at most 5,000 characters, author, whether it notified subscribers). Automatic ones are + private until a member publishes them or the status page publishes automatically. During a + maintenance window its components show maintenance, and their state changes are recorded without + alerts or automatic incidents. `monitor.incident` is a stored event and webhook type. +- **Status pages (M16.8).** `status_pages`: a slug (unique on the server; the default address is + `/status/{slug}/` on the Yuva server), name, logo (an uploaded image as in M13), accent colours for + light and dark, theme, languages (en, tr, de) with a default, time zone, visibility `public`, + `password` (argon2id, unlocking sets a 12-hour signed `HttpOnly` cookie on that host, the only cookie + a status page sets) or `members` (default address only, through the panel session), groups of + components (a service, dependency or check with a public name; components not listed stay private), + manual state overrides, `auto_publish`, and the inbox and e-mail channel that mail subscribers. The + page is rendered by the server with `html/template` and the server's en, tr and de catalogs, without + JavaScript: one inline stylesheet under a nonce, light and dark through `prefers-color-scheme`, every + state given in text as well as colour, `Accept-Language` or `?lang=` picking the language. It shows + each component's state, 90 days of uptime bars, response time for components backed by a check, + open incidents with their updates, maintenance and a history. A snapshot per page is rebuilt on any + change and every 60 s, so a view is a memory read; answers carry `Cache-Control: public, max-age=30` + and an ETag; no cookie, script, analytics or address in the request log. Next to the page (under + `/status/{slug}/` at the default address) the server serves + `/status.json` (state, components, open incidents and maintenance; `Access-Control-Allow-Origin: *`), + `/feed.rss`, `/feed.atom`, `/badge.svg`, `/embed` (the only path that may be framed) and + `/subscribe`. +- **Custom domains.** `status_page_domains` (hostname unique on the server, page, token, state + `pending`, `verified` or `failed`, checks, certificate state; 5 per page). A domain is verified when + it points at `YUVA_STATUS_CNAME` (the host of `YUVA_PUBLIC_URL` by default) by CNAME, or by A and + AAAA records equal to that host's for an apex, and `_yuva-status.` has a TXT record with + the domain's token. The TXT record ties the domain to its workspace, so a CNAME left behind after a + workspace moves away cannot be claimed by another workspace on the same server. The `status_domains` + job checks pending domains every 5 minutes for 7 days and verified ones hourly; one whose records + have been gone for 72 hours is no longer served and its certificate is deleted. A request to a + verified host reaches only that page's paths, never the panel or the API. + `YUVA_STATUS_TLS` chooses how certificates come about: + - `proxy` (default): the operator's proxy terminates TLS and forwards the host; `GET + /status-domains/check?domain=` answers `200` for a verified domain and `404` otherwise, which + is the contract of Caddy's on-demand TLS `ask` endpoint. + - `acme`: the server also listens on `YUVA_STATUS_TLS_ADDR` (`:8443`) for TLS and serves verified + status hostnames only, by SNI, with no default certificate. Certificates come from + `YUVA_ACME_DIRECTORY` (Let's Encrypt by default; `YUVA_ACME_EMAIL` optional) through + `golang.org/x/crypto/acme`, already a dependency. A `status_cert` job, queued by verification and + unique per hostname, orders the certificate; a TLS handshake never starts an order, so an unknown + name cannot spend the server's rate limits. HTTP-01 challenges are kept in `acme_challenges` and + answered at `/.well-known/acme-challenge/{token}` on the main listener for any host, so whichever + process the operator's port 80 reaches answers. Keys are ECDSA P-256, sealed under the master key in + `tls_certificates`, kept in each process's memory and reloaded on `NOTIFY`. Renewal starts at + two-thirds of a certificate's lifetime (Let's Encrypt's default lifetime falls to 64 days in + February 2027 and 45 in February 2028, and `x/crypto/acme` has no renewal-information support); + failures back off 1, 6 and 24 hours, under Let's Encrypt's 5 failed authorizations per hostname an + hour. The ACME account key is in `acme_accounts`, sealed. +- **Subscribers.** E-mail subscribers are a mailing list of the page's inbox, made with the page (not + public, not commercial), so double opt-in, consent events, the `/s/` and `/m/` pages, one-click + unsubscribe, suppression and opt-outs are M13's. The page's form is a plain POST with a sealed form + token, a honeypot and M13's per-address limits, and gives the same answer for every address. A notice + is a broadcast of kind `status` built from a fixed server template (the update, affected components, + a link) and sent by `broadcast_send` through the page's e-mail channel: it skips the design and review + steps but keeps suppression, the hosted approval gate and the daily cap. Members choose per update + whether it notifies (on by default for a new incident and its resolution). RSS 2.0 and Atom list + published incidents and maintenance. There are no public webhook subscriptions, because a form that + makes the server call any address is an SSRF and abuse amplifier; workspace webhooks get + `monitor.incident`. +- **Panel, API, MCP.** An API property opens Overview (requests, error rates and p95 over time, the + last 60 minutes by minute), Routes, a route's page (history, status codes, consumers, heatmap), + Consumers, Uptime, Alerts and Settings in the Analytics rail section; Dashboards sit at the top of that + section; a Monitoring section holds Services, Uptime checks, Incidents and Status pages. Owners, + admins and keys with `analytics:manage` manage properties, checks, services, dashboards and status + pages; agents read the API properties of their inboxes; services, checks, incidents and status pages + are readable by every member. MCP adds tools for slow and failing routes, a route's history, a + consumer's errors, alert state, service health and incidents, takes the new datasets in the query + tool, adds a `setup_api_monitoring` prompt and an entry in `get_setup_state`; consumer names are + `customer_content`. +- **Law.** Server-side records and OTLP spans read nothing from anyone's device, so ePrivacy Art. 5(3) + and PECR do not apply to them; the only value that can be personal data is `consumer`, which the host + chooses (the guide says to use accounts or API clients, and e-mail-shaped values are scrubbed), kept + in dimension rollups and erasable per consumer, for the host's legitimate interest in the security + and performance of its service (GDPR Recital 49, KVKK Art. 5(2)(f)). Countries are kept, addresses + never. App-side records leave the device, so they follow the property's `collection` and the app's + opt-out exactly like app events and are off until the host enables them. Status page subscribers are + M13 consent. CCPA and LGPD: nothing is sold or shared; no record concerns a child as such. +- **Hosting and scale.** `usage_counters.monitor_requests` and `YUVA_MONITOR_MONTHLY_REQUESTS` (per + workspace; unset: 10,000,000 with `YUVA_SIGNUP=open`, else no limit; operator override `yuva workspace + monitor-limit`), separate from analytics events because request volume is orders of magnitude above + page views. `YUVA_MONITOR=off` removes M16; `YUVA_ANALYTICS=off` removes both. The query layer sits + behind a `monitor.Store` interface as analytics does. Writes and rows grow with distinct keys per + minute and hour, not with requests: Postgres carries about 50,000 open core keys and 500,000 open + dimension keys across the server (about 5,000 and 50,000 row writes a second from the 10-second + upserts, HOT updates on `fillfactor 70` tables), and one process decodes tens of thousands of + records a second. It stops scaling when a single property needs more distinct combinations than the + caps allow (thousands of routes times thousands of consumers per hour) or the open keys across the + server pass those numbers; then a column store goes behind the `Store` interface without a contract + change. Uptime checks run from the server's one location. + ### Storage Attachments and raw e-mails in S3-compatible storage (R2, S3, MinIO) or on local disk for a @@ -1752,6 +2071,12 @@ now; per-channel limits can narrow them later. (25); salts are deleted two hours after their day. Members delete a property's data for a range or in full; a property, a consented visitor, a contact with linked visits and a deleted workspace take their analytics rows with them. Workspace `retention_days` does not apply to analytics. +- API monitoring keeps no row per request and no address: minute rows 8 days + (`YUVA_MONITOR_MINUTE_DAYS`), hours the property's `monitor_hour_days` (90), days its + `retention_months` (25); check results 30 days, check days, state changes and service history 400 + days. A consumer's rows and name are erased with `DELETE …/consumers/{id}`; a property, service, + check, status page or deleted workspace takes its rows with it. Status page subscribers are mailing + list subscriptions and follow M13's rules. Workspace `retention_days` does not apply. - Nothing is sent to third parties except what a channel is configured to send (SMTP, webhooks); the monthly GeoIP download from DB-IP fetches a file and sends nothing about anyone. @@ -1802,6 +2127,21 @@ now; per-channel limits can narrow them later. rows of that day, are created by the flusher (yesterday to the day after tomorrow in UTC when it starts, and any day it is about to write that has none) and by the `analytics_partitions` job, which also drops them; every other read, write and delete of those tables filters by `workspace_id`. + - API monitoring ingest names a property id that its API key must belong to, and app-side records + name the property's public key like analytics collection. A status page request finds its page by + `status_pages.slug` or `status_page_domains.hostname` (unique across the server), the Caddy `ask` + check reads `status_page_domains` by hostname, the TLS listener finds `tls_certificates` by + hostname, and `/.well-known/acme-challenge/` reads `acme_challenges` by token, all before the + workspace is known; everything after those lookups is scoped by the workspace they returned. + `acme_accounts` belongs to the server, not to a workspace. Whole day partitions of + `monitor_minute` and `monitor_check_results` and month partitions of `monitor_hour` and + `monitor_dim_hour` hold every workspace's rows and are made and dropped by the aggregator and the + `analytics_partitions` job; `monitor_checks_due`, `monitor_alerts`, `monitor_states` and + `status_domains` list due work across workspaces, and every read, write and delete after that + filters by `workspace_id`. + +Server (`api/`), the River bullet gains: "uptime checks, health pulls and certificate orders on their +own `monitoring` queue (see "API monitoring")". - A forwarding address names only its token: ingress finds the channel by `email_channels.forward_token` (unique across the server), like an address. A migration probe is recognised by its HMAC before the channel's workspace matters. diff --git a/docs/roadmap.md b/docs/roadmap.md index 74f4505f..4f0e233b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -260,3 +260,53 @@ without joining any visitor to a contact; Claude Code answers a traffic question a share link shows only its views; after consent and identify a contact's page lists their visits and without consent it never does; another workspace sees none of it; the load test meets the targets at 10M events a month. + +## M16 — API monitoring + +Status: planned; umbrella issue #63, decisions in `architecture.md` › API monitoring (M16). Ships in +eight steps; M16.1 first, then M16.2, M16.3 and M16.5 in parallel, then M16.4 and M16.6 (after M16.3), +then M16.7 (after M16.4) and M16.8 (after M16.7). The web SDK server helpers (#60, #61, #62) come +before M16.2. + +- **M16.1 Record and count:** API properties, the batched records endpoint, route inference and caps, + in-memory minute and hour aggregation, mergeable duration sketches, minute, hour and day tiers with + their folds, the `net/http` middleware in `sdk/go/analytics`, the API overview, routes and route + page, usage and limits, the guide. +- **M16.2 Every API stack:** OTLP/HTTP traces (protobuf and JSON), middleware on `useyuva/server` for + srvx, h3, Nitro and Nuxt, Hono, Express, Fastify and Next.js, app-side records from the Swift and + Kotlin modules with their store privacy answers, a guide per stack. +- **M16.3 Full analysis:** every core dimension and one extra dimension (consumer, country, app + version) as breakdown and filter, comparisons, Apdex, throughput, new and silent routes, response + sizes, consumers, deploy markers. +- **M16.4 Alerts, uptime, reports, MCP:** alert rules that fire and resolve once, uptime checks from the + server behind the address checks, the `uptime` dataset, weekly and monthly API reports, + conversations next to errors, MCP tools. +- **M16.5 Sharing, export, retention:** share links for API views, CSV and NDJSON exports of the + `api` and `uptime` datasets, per-tier retention, consumer erasure. +- **M16.6 Dashboards:** dashboard documents with variables, panels and thresholds over every dataset, + the editor, starter dashboards, import and export, dashboard share links, the Prometheus-compatible + endpoint for Grafana. +- **M16.7 Service health:** services, dependencies and instances from pushed or pulled health reports, + state from reports, checks and the API's own errors, state history, automatic incidents and their + alerts, health helpers in `sdk/go` and `useyuva/server`. +- **M16.8 Status pages:** server-rendered pages without JavaScript, components and groups, incidents + and maintenance with updates, 90-day uptime, subscribers through a mailing list, feeds, JSON, badge + and embed, custom domains verified by CNAME and TXT, certificates by ACME or by the operator's proxy. + +Accept: a Go service with the middleware, the same API on h3, Nitro, Hono and Express, and an +OpenTelemetry-instrumented service show identical routes and counts within a minute, with error rates +equal to the requests sent and p50, p95 and p99 within 1.1% of the exact values, from minute rows and, +the next day, from folded days with the same numbers; a raw path with an id never becomes a route and +404 probes stay in `(unmatched)`; Postgres holds no row per request and no IP address; a key without +`analytics:write` gets `403`; app-side records from the Swift and Kotlin samples appear next to the +server's for the same route and are not sent before consent on a consent-mode property; a route pushed +past its error-rate threshold raises one alert and one resolution, by e-mail, push and webhook; an +uptime check against a stopped endpoint alerts, and one aimed at a private or metadata address is +refused; stopping a sample service's dependency turns it degraded or down, opens one incident, turns +its component on a status page at `status.` (CNAME and TXT verified, HTTPS with a +certificate the server obtained from a test ACME directory) degraded and mails a confirmed subscriber +once, and resolving it updates the page, the feeds, the badge and the JSON; the page works with +JavaScript off; a dashboard and Grafana with the Yuva data source draw the same series as the fixed +views; an export matches the panel; Claude Code answers "which routes are failing" through the MCP +tools; another workspace sees none of it; the load test meets the targets at 300M records a month on +one property with 500 routes and 5,000 consumers. From 634d9d2c66bb6d7e570260e0d925f75cf89f79e9 Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:40:06 +0200 Subject: [PATCH 02/69] feat(api): API properties count their requests per route, with error rates and p50, p95 and p99 An analytics property can now be of kind api. Its host sends request records to POST /v1/analytics/properties/{id}/requests (1-1,000 a call, gzip, analytics:write); each process adds them up per UTC minute and local hour, with log-bucketed duration sketches, and writes cumulative rows under its own writer every 10 seconds, so Postgres holds no row per request and no address. Paths become route patterns with ids taken out, 404 and 405 probes count as (unmatched), and caps send new values to (other). The monitor_rollup job folds closed hours and finished local days, and the query operation answers dataset api from days, hours and minutes. YUVA_MONITOR, YUVA_MONITOR_MONTHLY_REQUESTS, YUVA_MONITOR_RECORDS_PER_MINUTE, YUVA_MONITOR_BUFFER_KEYS, YUVA_MONITOR_MINUTE_DAYS and yuva workspace monitor-limit set it up. Co-Authored-By: Claude Opus 5.5 --- api/cmd/yuva/main.go | 5 + api/cmd/yuva/operator.go | 35 +- api/internal/api/analytics_apps.go | 3 + api/internal/api/analytics_collect.go | 28 +- api/internal/api/analytics_goals.go | 6 + api/internal/api/analytics_jobs.go | 23 +- api/internal/api/analytics_properties.go | 80 +- api/internal/api/analytics_query.go | 14 + api/internal/api/monitor_jobs.go | 203 +++++ api/internal/api/monitor_query.go | 273 +++++++ api/internal/api/monitor_records.go | 410 ++++++++++ api/internal/api/monitor_test.go | 463 +++++++++++ api/internal/api/scopes.go | 1 + api/internal/api/server.go | 2 +- api/internal/config/config.go | 40 + api/internal/monitor/aggregate.go | 615 +++++++++++++++ api/internal/monitor/fold.go | 340 +++++++++ api/internal/monitor/maintain.go | 76 ++ api/internal/monitor/monitor.go | 274 +++++++ api/internal/monitor/pgstore.go | 475 ++++++++++++ api/internal/monitor/query.go | 722 ++++++++++++++++++ api/internal/monitor/route.go | 139 ++++ api/internal/monitor/route_test.go | 48 ++ api/internal/monitor/sketch.go | 223 ++++++ api/internal/monitor/sketch_test.go | 182 +++++ api/internal/oas/oas.gen.go | 707 +++++++++++++++-- api/internal/store/analytics.sql.gen.go | 66 +- api/internal/store/broadcasts.sql.gen.go | 9 +- .../store/migrations/00056_monitor.sql | 245 ++++++ api/internal/store/models.gen.go | 166 ++++ api/internal/store/monitor.sql.gen.go | 325 ++++++++ api/internal/store/queries/analytics.sql | 14 +- api/internal/store/queries/monitor.sql | 75 ++ api/internal/store/usage.sql.gen.go | 3 +- api/internal/store/workspaces.sql.gen.go | 18 +- openapi/openapi.yaml | 365 ++++++++- 36 files changed, 6516 insertions(+), 157 deletions(-) create mode 100644 api/internal/api/monitor_jobs.go create mode 100644 api/internal/api/monitor_query.go create mode 100644 api/internal/api/monitor_records.go create mode 100644 api/internal/api/monitor_test.go create mode 100644 api/internal/monitor/aggregate.go create mode 100644 api/internal/monitor/fold.go create mode 100644 api/internal/monitor/maintain.go create mode 100644 api/internal/monitor/monitor.go create mode 100644 api/internal/monitor/pgstore.go create mode 100644 api/internal/monitor/query.go create mode 100644 api/internal/monitor/route.go create mode 100644 api/internal/monitor/route_test.go create mode 100644 api/internal/monitor/sketch.go create mode 100644 api/internal/monitor/sketch_test.go create mode 100644 api/internal/store/migrations/00056_monitor.sql create mode 100644 api/internal/store/monitor.sql.gen.go create mode 100644 api/internal/store/queries/monitor.sql diff --git a/api/cmd/yuva/main.go b/api/cmd/yuva/main.go index ed71504a..35fd232f 100644 --- a/api/cmd/yuva/main.go +++ b/api/cmd/yuva/main.go @@ -54,6 +54,7 @@ const usageText = `usage: yuva workspace broadcasts limit --workspace --daily (0 removes the limit) yuva workspace broadcasts show --workspace yuva workspace analytics-limit --workspace (0 removes the limit) + yuva workspace monitor-limit --workspace (0 removes the limit) yuva analytics property create --workspace --name --domain ... [--inbox ] [--platform web|ios|android]... [--timezone ] (prints the property id and its public key, tab-separated) yuva analytics property list --workspace @@ -274,6 +275,8 @@ func serve(ctx context.Context, cfg config.Config, log *slog.Logger, args []stri } if !cfg.Analytics.On { log.Info("YUVA_ANALYTICS is off; analytics endpoints, the script and collection are not served") + } else if !cfg.Analytics.Monitor.On { + log.Info("YUVA_MONITOR is off; API properties and request records are not served") } if cfg.IngressAcceptV1 { log.Warn("YUVA_INGRESS_ACCEPT_V1 is set; /ingress/email accepts v1 signatures, which do not cover the envelope sender") @@ -332,6 +335,8 @@ func analyticsSettings(cfg config.Config) api.AnalyticsSettings { Off: !a.On, Buffer: a.Buffer, VisitCache: a.VisitCache, SaltGrace: a.SaltGrace, MonthlyEvents: a.MonthlyEvents, PropertyEventsPerMinute: a.PropertyEventsPerMinute, PropertiesPerWorkspace: a.PropertiesPerWorkspace, RawDays: a.RawDays, RollupTop: a.RollupTop, GeoIP: a.GeoIP, GeoIPDir: a.GeoIPDir, GeoIPFile: a.GeoIPFile, + Monitor: api.MonitorSettings{Off: !a.Monitor.On, MonthlyRequests: a.Monitor.MonthlyRequests, RecordsPerMinute: a.Monitor.RecordsPerMinute, + BufferKeys: a.Monitor.BufferKeys, MinuteDays: a.Monitor.MinuteDays}, } } diff --git a/api/cmd/yuva/operator.go b/api/cmd/yuva/operator.go index 0daccbf5..ffaf5ff5 100644 --- a/api/cmd/yuva/operator.go +++ b/api/cmd/yuva/operator.go @@ -34,7 +34,8 @@ const ( " yuva workspace broadcasts approve|deny|suspend --workspace [--reason ]\n" + " yuva workspace broadcasts limit --workspace --daily (0 removes the limit)\n" + " yuva workspace broadcasts show --workspace \n" + - " yuva workspace analytics-limit --workspace (0 removes the limit)" + " yuva workspace analytics-limit --workspace (0 removes the limit)\n" + + " yuva workspace monitor-limit --workspace (0 removes the limit)" analyticsUsage = "usage: yuva analytics property create --workspace --name --domain ... [--inbox ] [--platform web|ios|android]... [--timezone ]\n" + " yuva analytics property list --workspace \n" + " yuva analytics geoip-update" @@ -68,6 +69,9 @@ func operatorCLI(ctx context.Context, cfg config.Config, cmd string, args []stri if len(args) > 0 && args[0] == "analytics-limit" { return analyticsLimitCommand(ctx, st, srv, args[1:], os.Stderr) } + if len(args) > 0 && args[0] == "monitor-limit" { + return monitorLimitCommand(ctx, st, srv, args[1:], os.Stderr) + } return workspaceCommand(ctx, st, srv, args, os.Stderr) case "person": return personCommand(ctx, srv, args, os.Stderr) @@ -516,6 +520,35 @@ func analyticsLimitCommand(ctx context.Context, st *store.Store, srv *api.Server return nil } +func monitorLimitCommand(ctx context.Context, st *store.Store, srv *api.Server, args []string, stderr io.Writer) error { + fs := flag.NewFlagSet("workspace monitor-limit", flag.ContinueOnError) + fs.SetOutput(stderr) + workspace := fs.String("workspace", "", "workspace id or exact name") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New(workspaceUsage) + } + requests, err := strconv.ParseInt(fs.Arg(0), 10, 64) + if err != nil || requests < 0 { + return errors.New("the limit must be a number of requests, 0 for no limit") + } + ws, err := api.FindWorkspace(ctx, st, *workspace) + if err != nil { + return err + } + if err := srv.OperatorLimitMonitor(ctx, ws.ID, requests); err != nil { + return err + } + if requests == 0 { + fmt.Fprintf(stderr, "workspace %q (%s): API request records are not limited\n", ws.Name, ws.ID) + return nil + } + fmt.Fprintf(stderr, "workspace %q (%s): at most %d API request records a month\n", ws.Name, ws.ID, requests) + return nil +} + func analyticsCommand(ctx context.Context, st *store.Store, srv *api.Server, args []string, stdout, stderr io.Writer) error { if len(args) < 2 || args[0] != "property" { return errors.New(analyticsUsage) diff --git a/api/internal/api/analytics_apps.go b/api/internal/api/analytics_apps.go index 5bc426c6..2d9eb7cc 100644 --- a/api/internal/api/analytics_apps.go +++ b/api/internal/api/analytics_apps.go @@ -285,6 +285,9 @@ func (s *Server) SendAnalyticsEvents(ctx context.Context, req oas.SendAnalyticsE if err != nil { return nil, err } + if err := siteProperty(prop); err != nil { + return nil, err + } if req.Body == nil || len(req.Body.Events) == 0 || len(req.Body.Events) > maxAnalyticsBatch { return nil, errValidation("events must hold 1 to 100 events") } diff --git a/api/internal/api/analytics_collect.go b/api/internal/api/analytics_collect.go index 66cf5d15..9996bb96 100644 --- a/api/internal/api/analytics_collect.go +++ b/api/internal/api/analytics_collect.go @@ -72,6 +72,7 @@ type AnalyticsSettings struct { GeoIPFile string // GeoIPURL replaces DB-IP's download address, for tests. GeoIPURL func(month time.Time) string + Monitor MonitorSettings } func analyticsSettings(a AnalyticsSettings) AnalyticsSettings { @@ -99,6 +100,7 @@ func analyticsSettings(a AnalyticsSettings) AnalyticsSettings { if a.GeoIP == "" { a.GeoIP = analytics.GeoOff } + a.Monitor = monitorSettings(a.Monitor) return a } @@ -115,6 +117,7 @@ type analyticsState struct { props *propertyCache usage *usageCache limits *rateLimiter + monitor *monitorState } func (s *Server) newAnalytics(d Deps) *analyticsState { @@ -129,6 +132,9 @@ func (s *Server) newAnalytics(d Deps) *analyticsState { a.geo = &analytics.GeoIP{Mode: set.GeoIP, Dir: set.GeoIPDir, File: set.GeoIPFile, URL: set.GeoIPURL, Now: s.now} } a.flusher = &analytics.Flusher{Store: st, Buffer: a.buffer, Log: s.log, Now: s.now, AfterFlush: s.afterAnalyticsFlush} + if !d.Analytics.Monitor.Off { + a.monitor = s.newMonitor(d) + } if d.Hub != nil { d.Hub.OnInvalidate(a.invalidated) } @@ -136,6 +142,9 @@ func (s *Server) newAnalytics(d Deps) *analyticsState { } func (a *analyticsState) forget(propertyID uuid.UUID) { + if a.monitor != nil { + a.monitor.agg.Forget(propertyID) + } a.props.forget(propertyID) a.answers.forget(propertyID) a.resolver.Salts.Forget(propertyID) @@ -187,12 +196,23 @@ func (a *analyticsState) invalidated(payload string) { } } -// RunAnalytics writes collected events until ctx ends and then drains what is left; it returns at -// once when analytics is off. +// RunAnalytics writes collected events and request rows until ctx ends and then drains what is +// left; it returns at once when analytics is off. func (s *Server) RunAnalytics(ctx context.Context) { if s.analytics == nil { return } + if m := s.analytics.monitor; m != nil { + if err := m.agg.Prepare(ctx); err != nil { + s.log.ErrorContext(ctx, "monitor partitions", slog.Any("error", err)) + } + done := make(chan struct{}) + go func() { + defer close(done) + m.agg.Run(ctx) + }() + defer func() { <-done }() + } if err := s.analytics.flusher.Prepare(ctx); err != nil { s.log.ErrorContext(ctx, "analytics partitions", slog.Any("error", err)) } @@ -371,7 +391,7 @@ func (s *Server) analyticsUsage(ctx context.Context, ws uuid.UUID) (oas.Analytic out.LimitReachedAt = u.LimitReachedAt } } - return out, nil + return out, s.monitorUsage(ctx, ws, &out) } // afterAnalyticsFlush tells owners and the operator, once a month, that a workspace reached its cap. @@ -485,7 +505,7 @@ func (s *Server) analyticsCORS(w http.ResponseWriter, r *http.Request, next http writeProblem(w, errInternal) return } - if cp == nil { + if cp == nil || isAPIProperty(cp.prop) { writeProblem(w, errNotFound) return } diff --git a/api/internal/api/analytics_goals.go b/api/internal/api/analytics_goals.go index 455f905e..fa213fb9 100644 --- a/api/internal/api/analytics_goals.go +++ b/api/internal/api/analytics_goals.go @@ -193,6 +193,9 @@ func (s *Server) CreateAnalyticsGoal(ctx context.Context, req oas.CreateAnalytic if err := requireAnalyticsManager(p); err != nil { return err } + if err := siteProperty(prop); err != nil { + return err + } if err := f.validate(); err != nil { return err } @@ -397,6 +400,9 @@ func (s *Server) CreateAnalyticsFunnel(ctx context.Context, req oas.CreateAnalyt if err := requireAnalyticsManager(p); err != nil { return err } + if err := siteProperty(prop); err != nil { + return err + } name, err := trimmed(req.Body.Name, 1, 120, "name") if err != nil { return err diff --git a/api/internal/api/analytics_jobs.go b/api/internal/api/analytics_jobs.go index 7bd497d2..b804d8bb 100644 --- a/api/internal/api/analytics_jobs.go +++ b/api/internal/api/analytics_jobs.go @@ -91,10 +91,16 @@ func (s *Server) DeleteAnalytics(ctx context.Context, a AnalyticsDeleteArgs, bud } from, to = &f, &t } - p, err := s.purgeAnalyticsRaw(ctx, a.WorkspaceID, a.PropertyID, from, to, time.Now().Add(budget)) + deadline := time.Now().Add(budget) + p, err := s.purgeAnalyticsRaw(ctx, a.WorkspaceID, a.PropertyID, from, to, deadline) if err != nil { return false, err } + if p.Done { + if p.Done, err = s.purgeMonitor(ctx, a.WorkspaceID, a.PropertyID, from, to, deadline); err != nil { + return false, err + } + } if p.Done { switch { case a.Property: @@ -176,6 +182,9 @@ func (s *Server) purgeWorkspaceAnalytics(ctx context.Context, ws uuid.UUID, dead if err != nil || !p.Done { return false, err } + if done, err := s.purgeMonitor(ctx, ws, id, nil, nil, deadline); err != nil || !done { + return false, err + } } return true, nil } @@ -362,6 +371,7 @@ func (s *Server) addAnalyticsWorkers(workers *river.Workers) { river.AddWorker(workers, &analyticsExportWorker{s: s}) river.AddWorker(workers, &analyticsAlertsWorker{s: s}) river.AddWorker(workers, &analyticsReportsWorker{s: s}) + river.AddWorker(workers, &monitorRollupWorker{s: s}) } // rawFrom is the oldest day whose raw rows the partitions job keeps. @@ -379,6 +389,9 @@ func (s *Server) MaintainAnalytics(ctx context.Context) error { if err := s.analytics.store.EnsurePartitions(ctx, today.AddDate(0, 0, -partitionsBehind), today.AddDate(0, 0, partitionsAhead)); err != nil { return err } + if err := s.maintainMonitor(ctx, today); err != nil { + return err + } pg := &analytics.PGStore{Pool: s.st.Pool} dropped, err := pg.DropPartitionsBefore(ctx, s.rawFrom().AddDate(0, 0, -1)) if err != nil { @@ -439,6 +452,9 @@ func (s *Server) MaintainAnalytics(ctx context.Context) error { if _, err := s.st.DeleteOldAnalyticsIngestStats(ctx, store.DeleteOldAnalyticsIngestStatsParams{WorkspaceID: ws, Before: today.AddDate(0, 0, -analyticsStatsDays)}); err != nil { return err } + if err := s.trimMonitor(ctx, ws, today); err != nil { + return err + } } return nil } @@ -481,6 +497,11 @@ func (s *Server) ScanAnalyticsRollups(ctx context.Context) (int, error) { return queued, err } } + n, err := s.scanMonitorRollups(ctx, ws) + if err != nil { + return queued, err + } + queued += n stale, err := s.st.ListAnalyticsGoalRollupTodo(ctx, store.ListAnalyticsGoalRollupTodoParams{WorkspaceID: ws, Oldest: s.rawFrom()}) if err != nil { return queued, err diff --git a/api/internal/api/analytics_properties.go b/api/internal/api/analytics_properties.go index 3de3cd1d..77e916d2 100644 --- a/api/internal/api/analytics_properties.go +++ b/api/internal/api/analytics_properties.go @@ -26,12 +26,16 @@ const ( analyticsKeyOverlap = 24 * time.Hour defaultRetentionMonths = 25 maxAnalyticsDeleteRange = 400 + defaultApdexMS = 500 + defaultMonitorHourDays = 90 ) var ( errPropertyGone = problem(http.StatusNotFound, "not_found", "no such analytics property") errPropertyLimit = problem(http.StatusForbidden, "property_limit", "this workspace has as many analytics properties as the server allows (YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE)") errPropertyConfirm = problem(http.StatusBadRequest, "confirmation_mismatch", "type the property's name exactly to delete it") + errSiteOnly = problem(http.StatusUnprocessableEntity, "property_kind", "this is an API property; it takes request records and dataset api") + errAPIOnly = problem(http.StatusUnprocessableEntity, "property_kind", "this is a site property; request records and dataset api need an API property") analyticsDomainPattern = regexp.MustCompile(`^(\*\.)?([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`) currencyPattern = regexp.MustCompile(`^[A-Z]{3}$`) ) @@ -79,6 +83,18 @@ func visibleProperty(ctx context.Context, q *store.Queries, p principal, id uuid return prop, nil } +func isAPIProperty(prop store.AnalyticsProperty) bool { + return prop.Kind == string(oas.AnalyticsKindApi) +} + +// siteProperty refuses an API property where only sites have the feature. +func siteProperty(prop store.AnalyticsProperty) error { + if isAPIProperty(prop) { + return errSiteOnly + } + return nil +} + func requireAnalyticsManager(p principal) error { if p.isKey() && p.limited { return errForbidden @@ -94,7 +110,8 @@ func pathRules(raw []byte) []analytics.PathRule { func (s *Server) propertyBody(prop store.AnalyticsProperty, visitorsToday int64) oas.AnalyticsProperty { out := oas.AnalyticsProperty{ - Id: prop.ID, InboxId: prop.InboxID, Name: prop.Name, PublicKey: prop.PublicKey, Timezone: prop.Timezone, Currency: prop.Currency, + Id: prop.ID, Kind: oas.AnalyticsPropertyKind(prop.Kind), InboxId: prop.InboxID, Name: prop.Name, PublicKey: prop.PublicKey, Timezone: prop.Timezone, Currency: prop.Currency, + ApdexMs: prop.ApdexMs, MonitorHourDays: prop.MonitorHourDays, Collection: oas.AnalyticsCollection(prop.Collection), Audience: oas.AnalyticsAudience(prop.Audience), Domains: prop.Domains, ExcludedPaths: prop.ExcludedPaths, TrackedProps: prop.TrackedProps, RetentionMonths: prop.RetentionMonths, VisitorsToday: visitorsToday, FirstEventAt: prop.FirstEventAt, LastEventAt: prop.LastEventAt, CreatedAt: prop.CreatedAt, UpdatedAt: prop.UpdatedAt, @@ -148,6 +165,7 @@ func propertyLocation(tz string) *time.Location { // propertyFields are a property's settings while they are checked. type propertyFields struct { + kind string inboxID *uuid.UUID name string domains []string @@ -162,13 +180,16 @@ type propertyFields struct { pathRules []analytics.PathRule trackedProps []string retention *int32 + apdexMS int32 + hourDays int32 } func fieldsOf(prop store.AnalyticsProperty) propertyFields { return propertyFields{ - inboxID: prop.InboxID, name: prop.Name, domains: prop.Domains, platforms: prop.Platforms, timezone: prop.Timezone, currency: prop.Currency, + kind: prop.Kind, inboxID: prop.InboxID, name: prop.Name, domains: prop.Domains, platforms: prop.Platforms, timezone: prop.Timezone, currency: prop.Currency, collection: prop.Collection, consentFeatures: prop.ConsentFeatures, audience: prop.Audience, excludedPaths: prop.ExcludedPaths, excludedIPs: prop.ExcludedIps, pathRules: pathRules(prop.PathRules), trackedProps: prop.TrackedProps, retention: prop.RetentionMonths, + apdexMS: prop.ApdexMs, hourDays: prop.MonitorHourDays, } } @@ -210,7 +231,8 @@ func (f *propertyFields) validate() error { if f.name, err = trimmed(f.name, 1, 100, "name"); err != nil { return err } - if len(f.domains) < 1 || len(f.domains) > 50 { + api := f.kind == string(oas.AnalyticsKindApi) + if (len(f.domains) < 1 && !api) || len(f.domains) > 50 { return errValidation("domains must list 1 to 50 host names") } for i, d := range f.domains { @@ -221,9 +243,12 @@ func (f *propertyFields) validate() error { f.domains[i] = n } f.domains = uniqueStrings(f.domains) - if len(f.platforms) == 0 { + if len(f.platforms) == 0 && !api { return errValidation("platforms must name at least one platform") } + if api && slices.Contains(f.platforms, string(oas.AnalyticsPlatformWeb)) { + return errValidation("an API property's platforms are the apps that send app-side records, ios and android") + } for _, v := range f.platforms { if !oas.AnalyticsPlatform(v).Valid() { return errValidation("unknown platform " + strconv.Quote(v)) @@ -298,6 +323,15 @@ func (f *propertyFields) validate() error { if f.retention != nil && (*f.retention < 1 || *f.retention > 120) { return errValidation("retention_months must be 1 to 120, or null to keep rollups forever") } + if api && (len(f.consentFeatures) > 0 || len(f.trackedProps) > 0 || f.audience != string(oas.AnalyticsAudienceGeneral)) { + return errValidation("an API property takes no consent features, tracked props or audience") + } + if f.apdexMS < 1 || f.apdexMS > 60000 { + return errValidation("apdex_ms must be 1 to 60000") + } + if f.hourDays < 7 || f.hourDays > 400 { + return errValidation("monitor_hour_days must be 7 to 400") + } return nil } @@ -388,6 +422,12 @@ func (s *Server) applyPropertyBody(ctx context.Context, q *store.Queries, p prin f.retention = new(b.RetentionMonths.MustGet()) } } + if b.ApdexMs != nil { + f.apdexMS = *b.ApdexMs + } + if b.MonitorHourDays != nil { + f.hourDays = *b.MonitorHourDays + } return f.validate() } @@ -428,9 +468,22 @@ func (s *Server) CreateAnalyticsProperty(ctx context.Context, req oas.CreateAnal } b := req.Body f := propertyFields{ - name: b.Name, timezone: "UTC", currency: "EUR", collection: string(oas.AnalyticsCollectionAnonymous), audience: string(oas.AnalyticsAudienceGeneral), - platforms: []string{string(oas.AnalyticsPlatformWeb)}, consentFeatures: []string{}, excludedPaths: []string{}, excludedIPs: []netip.Prefix{}, - pathRules: []analytics.PathRule{}, trackedProps: []string{}, retention: new(int32(defaultRetentionMonths)), + kind: string(oas.AnalyticsKindSite), name: b.Name, timezone: "UTC", currency: "EUR", collection: string(oas.AnalyticsCollectionAnonymous), + audience: string(oas.AnalyticsAudienceGeneral), platforms: []string{string(oas.AnalyticsPlatformWeb)}, consentFeatures: []string{}, + excludedPaths: []string{}, excludedIPs: []netip.Prefix{}, pathRules: []analytics.PathRule{}, trackedProps: []string{}, + retention: new(int32(defaultRetentionMonths)), apdexMS: defaultApdexMS, hourDays: defaultMonitorHourDays, + } + if b.Kind != nil { + if !b.Kind.Valid() { + return nil, errValidation("kind must be site or api") + } + f.kind = string(*b.Kind) + } + if f.kind == string(oas.AnalyticsKindApi) { + if s.analytics.monitor == nil { + return nil, errValidation("API monitoring is off on this server (YUVA_MONITOR=off)") + } + f.platforms = []string{} } var out store.AnalyticsProperty err := s.st.InTx(ctx, func(q *store.Queries) error { @@ -444,7 +497,8 @@ func (s *Server) CreateAnalyticsProperty(ctx context.Context, req oas.CreateAnal upd := oas.AnalyticsPropertyUpdate{ Domains: &b.Domains, Platforms: b.Platforms, Timezone: b.Timezone, Currency: b.Currency, Collection: b.Collection, Audience: b.Audience, ConsentFeatures: b.ConsentFeatures, ExcludedPaths: b.ExcludedPaths, ExcludedIps: b.ExcludedIps, - PathRules: b.PathRules, TrackedProps: b.TrackedProps, RetentionMonths: b.RetentionMonths, + PathRules: b.PathRules, TrackedProps: b.TrackedProps, RetentionMonths: b.RetentionMonths, ApdexMs: b.ApdexMs, + MonitorHourDays: b.MonitorHourDays, } if err := s.applyPropertyBody(ctx, q, p, &f, upd); err != nil { return err @@ -465,9 +519,9 @@ func (s *Server) CreateAnalyticsProperty(ctx context.Context, req oas.CreateAnal createdBy = &p.memberID } out, err = q.CreateAnalyticsProperty(ctx, store.CreateAnalyticsPropertyParams{ - ID: id, WorkspaceID: p.workspaceID, InboxID: f.inboxID, Name: f.name, PublicKey: newAnalyticsKey(), Domains: f.domains, - Platforms: f.platforms, Timezone: f.timezone, Currency: f.currency, Collection: f.collection, ConsentFeatures: nonNil(f.consentFeatures), - Audience: f.audience, ExcludedPaths: nonNil(f.excludedPaths), ExcludedIps: nonNil(f.excludedIPs), PathRules: mustJSON(nonNil(f.pathRules)), + ID: id, WorkspaceID: p.workspaceID, InboxID: f.inboxID, Kind: f.kind, Name: f.name, PublicKey: newAnalyticsKey(), Domains: nonNil(f.domains), + Platforms: nonNil(f.platforms), Timezone: f.timezone, Currency: f.currency, Collection: f.collection, ConsentFeatures: nonNil(f.consentFeatures), + ApdexMs: f.apdexMS, MonitorHourDays: f.hourDays, Audience: f.audience, ExcludedPaths: nonNil(f.excludedPaths), ExcludedIps: nonNil(f.excludedIPs), PathRules: mustJSON(nonNil(f.pathRules)), TrackedProps: nonNil(f.trackedProps), RetentionMonths: f.retention, VisitorKey: s.secrets.Seal(analytics.NewSalt(), analyticsSecretContext(p.workspaceID, id)), CreatedBy: createdBy, Now: s.now(), }) @@ -525,10 +579,10 @@ func (s *Server) UpdateAnalyticsProperty(ctx context.Context, req oas.UpdateAnal } } out, err = q.UpdateAnalyticsProperty(ctx, store.UpdateAnalyticsPropertyParams{ - WorkspaceID: p.workspaceID, ID: cur.ID, InboxID: f.inboxID, Name: f.name, Domains: f.domains, Platforms: f.platforms, + WorkspaceID: p.workspaceID, ID: cur.ID, InboxID: f.inboxID, Name: f.name, Domains: nonNil(f.domains), Platforms: nonNil(f.platforms), Timezone: f.timezone, Currency: f.currency, Collection: f.collection, ConsentFeatures: nonNil(f.consentFeatures), Audience: f.audience, ExcludedPaths: nonNil(f.excludedPaths), ExcludedIps: nonNil(f.excludedIPs), PathRules: mustJSON(nonNil(f.pathRules)), - TrackedProps: nonNil(f.trackedProps), RetentionMonths: f.retention, Now: s.now(), + TrackedProps: nonNil(f.trackedProps), RetentionMonths: f.retention, ApdexMs: f.apdexMS, MonitorHourDays: f.hourDays, Now: s.now(), }) if err != nil { return err diff --git a/api/internal/api/analytics_query.go b/api/internal/api/analytics_query.go index 2fb6b202..655c8243 100644 --- a/api/internal/api/analytics_query.go +++ b/api/internal/api/analytics_query.go @@ -140,6 +140,8 @@ func analyticsRange(r oas.AnalyticsRange, today time.Time) (time.Time, time.Time return month.AddDate(0, -1, 0), month.AddDate(0, 0, -1), nil case oas.AnalyticsThisYear: return time.Date(y, 1, 1, 0, 0, 0, 0, time.UTC), today, nil + case oas.AnalyticsLast15Minutes, oas.AnalyticsLastHour, oas.AnalyticsLast6Hours, oas.AnalyticsLast24Hours: + return time.Time{}, time.Time{}, errValidation("the periods 15m, 1h, 6h and 24h need dataset api") } return time.Time{}, time.Time{}, errValidation("unknown range.period") } @@ -332,6 +334,9 @@ func (s *Server) QueryAnalytics(ctx context.Context, req oas.QueryAnalyticsReque if err != nil { return nil, err } + if isAPIProperty(prop) || (req.Body.Dataset != nil && *req.Body.Dataset == oas.AnalyticsDatasetApi) { + return s.queryMonitor(ctx, prop, *req.Body, req.Params.Format != nil && *req.Params.Format == oas.AnalyticsFormatCSV) + } ar, err := s.analyticsRequest(ctx, prop, *req.Body) if err != nil { return nil, err @@ -364,6 +369,12 @@ type analyticsRequest struct { func (s *Server) analyticsRequest(ctx context.Context, prop store.AnalyticsProperty, b oas.AnalyticsQuery) (analyticsRequest, error) { var ar analyticsRequest + if err := siteProperty(prop); err != nil { + return ar, err + } + if b.Dataset != nil && *b.Dataset == oas.AnalyticsDatasetApi { + return ar, errAPIOnly + } ws := prop.WorkspaceID loc := propertyLocation(prop.Timezone) today := analytics.LocalDay(s.now(), loc) @@ -674,6 +685,9 @@ func (s *Server) GetAnalyticsRealtime(ctx context.Context, req oas.GetAnalyticsR if err != nil { return nil, err } + if err := siteProperty(prop); err != nil { + return nil, err + } out, err := s.analyticsRealtime(ctx, prop) if err != nil { return nil, err diff --git a/api/internal/api/monitor_jobs.go b/api/internal/api/monitor_jobs.go new file mode 100644 index 00000000..a5a1cfc4 --- /dev/null +++ b/api/internal/api/monitor_jobs.go @@ -0,0 +1,203 @@ +package api + +import ( + "context" + "log/slog" + "time" + "uuid" + + "github.com/riverqueue/river" + + "github.com/productdevbook/yuva/api/internal/jobs" + "github.com/productdevbook/yuva/api/internal/monitor" + "github.com/productdevbook/yuva/api/internal/store" +) + +const ( + monitorDeleteBatch = 50_000 + // maxMonitorHourDays is the longest monitor_hour_days; hour partitions older than that and a + // month are dropped. + maxMonitorHourDays = 400 +) + +// MonitorRollupArgs folds an API property's closed hours and finished days. +type MonitorRollupArgs struct { + WorkspaceID uuid.UUID `json:"workspace_id"` + PropertyID uuid.UUID `json:"property_id"` +} + +func (MonitorRollupArgs) Kind() string { return "monitor_rollup" } + +func (MonitorRollupArgs) InsertOpts() river.InsertOpts { + return river.InsertOpts{Queue: jobs.AnalyticsQueue, UniqueOpts: river.UniqueOpts{ByArgs: true, ByState: analyticsRollupStates}} +} + +type monitorRollupWorker struct { + river.WorkerDefaults[MonitorRollupArgs] + s *Server +} + +func (w *monitorRollupWorker) Timeout(*river.Job[MonitorRollupArgs]) time.Duration { + return analyticsRollupTimeout +} + +func (w *monitorRollupWorker) Work(ctx context.Context, job *river.Job[MonitorRollupArgs]) error { + return w.s.RollupMonitor(ctx, job.Args.WorkspaceID, job.Args.PropertyID) +} + +// RollupMonitor folds a property's hours that closed and its local days that ended. +func (s *Server) RollupMonitor(ctx context.Context, ws, propertyID uuid.UUID) error { + m, err := s.monitorOn() + if err != nil { + return nil + } + prop, err := s.st.GetAnalyticsProperty(ctx, store.GetAnalyticsPropertyParams{WorkspaceID: ws, ID: propertyID}) + if store.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + hours, days, err := m.store.Rollup(ctx, ws, propertyID, propertyLocation(prop.Timezone), s.now()) + if err != nil { + return err + } + if hours+days > 0 { + s.analytics.answers.forget(propertyID) + } + _ = river.RecordOutput(ctx, map[string]int{"hours": hours, "days": days}) + return nil +} + +// scanMonitorRollups queues a fold for every API property of the workspace with an hour that closed +// and is not folded, or a finished day that is not. +func (s *Server) scanMonitorRollups(ctx context.Context, ws uuid.UUID) (int, error) { + if s.analytics.monitor == nil { + return 0, nil + } + ids, err := s.st.ListMonitorRollupProperties(ctx, store.ListMonitorRollupPropertiesParams{WorkspaceID: ws, Before: s.now().Add(-time.Hour - monitor.FoldAfter)}) + if err != nil || len(ids) == 0 { + return 0, err + } + batch := make([]river.InsertManyParams, len(ids)) + for i, id := range ids { + batch[i] = river.InsertManyParams{Args: MonitorRollupArgs{WorkspaceID: ws, PropertyID: id}} + } + if _, err := s.jobs.InsertMany(ctx, batch); err != nil { + return 0, err + } + return len(batch), nil +} + +// maintainMonitor keeps minute partitions from yesterday to a week ahead and hour partitions for +// this month and the next, drops minute partitions past YUVA_MONITOR_MINUTE_DAYS and hour partitions +// past the longest hour retention. +func (s *Server) maintainMonitor(ctx context.Context, today time.Time) error { + m := s.analytics.monitor + if m == nil { + return nil + } + if err := m.store.EnsureMinutePartitions(ctx, today.AddDate(0, 0, -1), today.AddDate(0, 0, partitionsAhead)); err != nil { + return err + } + if err := m.store.EnsureHourPartitions(ctx, today, today.AddDate(0, 1, 0)); err != nil { + return err + } + month := time.Date(today.Year(), today.Month(), 1, 0, 0, 0, 0, time.UTC) + dropped, err := m.store.DropPartitionsBefore(ctx, today.AddDate(0, 0, 1-m.settings.MinuteDays), month.AddDate(0, 0, -maxMonitorHourDays).AddDate(0, -1, 0)) + if err != nil { + return err + } + if len(dropped) > 0 { + s.log.InfoContext(ctx, "monitor partitions dropped", slog.Int("count", len(dropped))) + } + return nil +} + +// trimMonitor deletes an API property's hours past its monitor_hour_days and its days past its +// retention_months, and ingest counts past 90 days. +func (s *Server) trimMonitor(ctx context.Context, ws uuid.UUID, today time.Time) error { + if s.analytics.monitor == nil { + return nil + } + props, err := s.st.ListMonitorRetention(ctx, ws) + if err != nil { + return err + } + drain := func(trim func() (int64, error)) error { + for { + n, err := trim() + if err != nil || n < 1000 { + return err + } + } + } + for _, p := range props { + loc := propertyLocation(p.Timezone) + hourBefore := monitor.DayStart(monitor.LocalDay(s.now(), loc).AddDate(0, 0, -int(p.MonitorHourDays)), loc) + if err := drain(func() (int64, error) { + return s.st.TrimMonitorHour(ctx, store.TrimMonitorHourParams{WorkspaceID: ws, PropertyID: p.ID, Before: hourBefore}) + }); err != nil { + return err + } + if err := drain(func() (int64, error) { + return s.st.TrimMonitorDimHour(ctx, store.TrimMonitorDimHourParams{WorkspaceID: ws, PropertyID: p.ID, Before: hourBefore}) + }); err != nil { + return err + } + if p.RetentionMonths == nil { + continue + } + dayBefore := monitor.LocalDay(s.now(), loc).AddDate(0, -int(*p.RetentionMonths), 0) + if err := drain(func() (int64, error) { + return s.st.TrimMonitorDay(ctx, store.TrimMonitorDayParams{WorkspaceID: ws, PropertyID: p.ID, Before: dayBefore}) + }); err != nil { + return err + } + if err := drain(func() (int64, error) { + return s.st.TrimMonitorDimDay(ctx, store.TrimMonitorDimDayParams{WorkspaceID: ws, PropertyID: p.ID, Before: dayBefore}) + }); err != nil { + return err + } + if err := drain(func() (int64, error) { + return s.st.TrimMonitorRollupHours(ctx, store.TrimMonitorRollupHoursParams{WorkspaceID: ws, PropertyID: p.ID, Before: monitor.DayStart(dayBefore, loc)}) + }); err != nil { + return err + } + } + _, err = s.st.DeleteOldMonitorIngestStats(ctx, store.DeleteOldMonitorIngestStatsParams{WorkspaceID: ws, Before: today.AddDate(0, 0, -analyticsStatsDays)}) + return err +} + +// purgeMonitor deletes a property's request rows: of the local days from to to, or all of them. +func (s *Server) purgeMonitor(ctx context.Context, ws, propertyID uuid.UUID, from, to *time.Time, deadline time.Time) (bool, error) { + var fromAt, toAt *time.Time + if from != nil { + prop, err := s.st.Queries.GetAnalyticsProperty(ctx, store.GetAnalyticsPropertyParams{WorkspaceID: ws, ID: propertyID}) + if store.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + loc := propertyLocation(prop.Timezone) + fromAt, toAt = new(monitor.DayStart(*from, loc)), new(monitor.DayStart(to.AddDate(0, 0, 1), loc)) + } + for { + n, err := s.st.DeleteMonitorMinutes(ctx, store.DeleteMonitorMinutesParams{WorkspaceID: ws, PropertyID: propertyID, FromAt: fromAt, ToAt: toAt, MaxRows: monitorDeleteBatch}) + if err != nil { + return false, err + } + if n < monitorDeleteBatch { + break + } + if time.Now().After(deadline) { + return false, nil + } + } + if from == nil { + return true, s.st.DeleteMonitorRollups(ctx, store.DeleteMonitorRollupsParams{WorkspaceID: ws, PropertyID: propertyID}) + } + return true, s.st.DeleteMonitorRange(ctx, store.DeleteMonitorRangeParams{WorkspaceID: ws, PropertyID: propertyID, FromAt: *fromAt, ToAt: *toAt, + FromDay: *from, ToDay: *to}) +} diff --git a/api/internal/api/monitor_query.go b/api/internal/api/monitor_query.go new file mode 100644 index 00000000..4c4dd162 --- /dev/null +++ b/api/internal/api/monitor_query.go @@ -0,0 +1,273 @@ +package api + +import ( + "bytes" + "cmp" + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "slices" + "strconv" + "strings" + "time" + + openapi_types "github.com/oapi-codegen/runtime/types" + + "github.com/productdevbook/yuva/api/internal/analytics" + "github.com/productdevbook/yuva/api/internal/monitor" + "github.com/productdevbook/yuva/api/internal/oas" + "github.com/productdevbook/yuva/api/internal/store" +) + +const monitorOpenTTL = 5 * time.Second + +var monitorPeriods = map[oas.AnalyticsRangePeriod]time.Duration{ + oas.AnalyticsLast15Minutes: 15 * time.Minute, oas.AnalyticsLastHour: time.Hour, oas.AnalyticsLast6Hours: 6 * time.Hour, + oas.AnalyticsLast24Hours: 24 * time.Hour, +} + +// monitorRequest is an api query checked and turned into the property's query and its page. +type monitorRequest struct { + q monitor.Query + order []analytics.Order + limit int + offset int +} + +func (s *Server) monitorRequest(prop store.AnalyticsProperty, b oas.AnalyticsQuery) (monitorRequest, error) { + var mr monitorRequest + m := s.analytics.monitor + if b.GoalId != nil || b.Prop != nil || b.SegmentId != nil || b.Compare != nil || (b.Insights != nil && *b.Insights) { + return mr, errValidation("the api dataset takes no goal_id, prop, segment_id, compare or insights") + } + loc, now := propertyLocation(prop.Timezone), s.now() + q := monitor.Query{WorkspaceID: prop.WorkspaceID, PropertyID: prop.ID, Location: loc, Now: now, + MinuteFrom: monitor.LocalDay(now, time.UTC).AddDate(0, 0, 1-m.settings.MinuteDays)} + if d, ok := monitorPeriods[derefPeriod(b.Range.Period)]; ok && b.Range.Period != nil { + if b.Range.From != nil || b.Range.To != nil { + return mr, errValidation("send either range.period, or range.from and range.to") + } + q.End = now.Truncate(time.Minute).Add(time.Minute) + q.Start = q.End.Add(-d) + } else { + from, to, err := analyticsRange(b.Range, monitor.LocalDay(now, loc)) + if err != nil { + return mr, err + } + q.Start, q.End, q.Days = monitor.DayStart(from, loc), monitor.DayStart(to.AddDate(0, 0, 1), loc), true + } + for _, metric := range b.Metrics { + if slices.Contains(q.Metrics, string(metric)) { + return mr, errValidation("metric " + string(metric) + " is listed twice") + } + q.Metrics = append(q.Metrics, string(metric)) + } + if b.Dimensions != nil { + for _, d := range *b.Dimensions { + q.Dimensions = append(q.Dimensions, string(d)) + } + } + if b.Filters != nil { + for _, f := range *b.Filters { + if len(f.Values) == 0 || len(f.Values) > 20 { + return mr, errValidation("a filter has 1 to 20 values") + } + if f.Prop != nil { + return mr, errValidation("the api dataset has no props") + } + q.Filters = append(q.Filters, monitor.Filter{Dimension: string(f.Dimension), Op: string(f.Op), Values: f.Values}) + } + } + first := "requests" + if !slices.Contains(q.Metrics, first) && len(q.Metrics) > 0 { + first = q.Metrics[0] + } + mr.order = []analytics.Order{{Metric: first, Desc: true}} + if b.OrderBy != nil && len(*b.OrderBy) > 0 { + mr.order = nil + for _, o := range *b.OrderBy { + if !slices.Contains(monitor.Metrics, string(o.Metric)) { + return mr, errValidation("the api dataset orders by its own metrics, not " + string(o.Metric)) + } + mr.order = append(mr.order, analytics.Order{Metric: string(o.Metric), Desc: o.Dir == nil || *o.Dir == oas.AnalyticsDesc}) + } + } + mr.limit = defaultAnalyticsLimit + if b.Limit != nil { + mr.limit = *b.Limit + } + if mr.limit < 1 || mr.limit > 1000 { + return mr, errValidation("limit must be 1 to 1000") + } + if b.Cursor != nil && *b.Cursor != "" { + raw, err := base64.RawURLEncoding.DecodeString(*b.Cursor) + if err == nil { + mr.offset, err = strconv.Atoi(string(raw)) + } + if err != nil || mr.offset < 0 { + return mr, errValidation("cursor is not one this query returned") + } + } + mr.q = q + return mr, nil +} + +func derefPeriod(p *oas.AnalyticsRangePeriod) oas.AnalyticsRangePeriod { + if p == nil { + return "" + } + return *p +} + +func monitorQueryError(err error) error { + if errors.Is(err, monitor.ErrMinuteRange) { + return problem(http.StatusUnprocessableEntity, "range_too_large", err.Error()) + } + return analyticsQueryError(err) +} + +func apiMetricsBody(a *monitor.Agg, minutes float64, names []string) oas.AnalyticsMetrics { + var m oas.AnalyticsMetrics + for _, n := range names { + v := a.Metric(n, minutes) + switch n { + case "requests": + m.Requests = &v + case "requests_per_minute": + m.RequestsPerMinute = &v + case "errors": + m.Errors = &v + case "error_rate": + m.ErrorRate = &v + case "client_errors": + m.ClientErrors = &v + case "client_error_rate": + m.ClientErrorRate = &v + case "p50": + m.P50 = &v + case "p95": + m.P95 = &v + case "p99": + m.P99 = &v + case "avg_duration": + m.AvgDuration = &v + } + } + return m +} + +// sortMonitorRows orders rows by the order's metrics; with a time grain, in time order first. +func sortMonitorRows(rows []monitor.Row, dims []string, order []analytics.Order) { + if timeOnly(dims) { + return + } + slices.SortStableFunc(rows, func(a, b monitor.Row) int { + for _, o := range order { + x, y := a.Agg.Metric(o.Metric, a.Minutes), b.Agg.Metric(o.Metric, b.Minutes) + if c := cmp.Compare(x, y); c != 0 { + if o.Desc { + return -c + } + return c + } + } + return strings.Compare(strings.Join(a.Dims, "\x1f"), strings.Join(b.Dims, "\x1f")) + }) + if ti := timeIndex(dims); ti >= 0 { + slices.SortStableFunc(rows, func(a, b monitor.Row) int { return strings.Compare(a.Dims[ti], b.Dims[ti]) }) + } +} + +// runMonitorQuery answers from the cache or the store, at most four queries at a time per +// workspace like every analytics query. Answers that read minute rows or reach into the current +// minute live five seconds, others until a fold moves the property's rollup version. +func (s *Server) runMonitorQuery(ctx context.Context, prop store.AnalyticsProperty, q monitor.Query) (monitor.Result, error) { + a, m := s.analytics, s.analytics.monitor + now := q.Now + keyQ := q + keyQ.Now, keyQ.Location = time.Time{}, nil + key, _ := json.Marshal(keyQ) + cacheKey := prop.ID.String() + "|" + strconv.Itoa(int(prop.RollupVersion)) + "|api|" + prop.Timezone + "|" + string(key) + if v, ok := a.answers.get(cacheKey, now); ok { + return v.(monitor.Result), nil + } + release, err := a.slots.acquire(ctx, prop.WorkspaceID) + if err != nil { + return monitor.Result{}, err + } + defer release() + res, err := m.store.Query(ctx, q) + if err != nil { + return res, monitorQueryError(err) + } + ttl := analyticsFinishedTTL + if res.ServedFrom != monitor.ServedRollups || q.End.After(now) { + ttl = monitorOpenTTL + } + a.answers.put(cacheKey, res, now, ttl) + return res, nil +} + +func (s *Server) queryMonitor(ctx context.Context, prop store.AnalyticsProperty, b oas.AnalyticsQuery, csv bool) (oas.QueryAnalyticsResponseObject, error) { + if s.analytics.monitor == nil { + return nil, errNotFound + } + if !isAPIProperty(prop) { + return nil, errAPIOnly + } + if b.Dataset == nil || *b.Dataset != oas.AnalyticsDatasetApi { + return nil, errSiteOnly + } + mr, err := s.monitorRequest(prop, b) + if err != nil { + return nil, err + } + res, err := s.runMonitorQuery(ctx, prop, mr.q) + if err != nil { + return nil, err + } + q := mr.q + rows := slices.Clone(res.Rows) + sortMonitorRows(rows, q.Dimensions, mr.order) + next := "" + if !timeOnly(q.Dimensions) { + offset := min(mr.offset, len(rows)) + end := min(offset+mr.limit, len(rows)) + if end < len(rows) { + next = base64.RawURLEncoding.EncodeToString([]byte(strconv.Itoa(end))) + } + rows = rows[offset:end] + } + out := oas.AnalyticsResult{ + Rows: make([]oas.AnalyticsRow, len(rows)), Totals: apiMetricsBody(&res.Totals, res.Minutes, q.Metrics), + Meta: oas.AnalyticsResultMeta{ + From: openapi_types.Date{Time: monitor.LocalDay(q.Start, q.Location)}, To: openapi_types.Date{Time: monitor.LocalDay(q.End.Add(-time.Nanosecond), q.Location)}, + Timezone: prop.Timezone, ServedFrom: oas.AnalyticsResultMetaServedFrom(res.ServedFrom), RawWindowFrom: openapi_types.Date{Time: q.MinuteFrom}, + }, + } + if res.Totals.Sampled > 0 { + out.Meta.Estimated = new(true) + } + if len(res.Caps) > 0 { + caps := make([]oas.AnalyticsResultMetaCapsReached, len(res.Caps)) + for i, c := range res.Caps { + caps[i] = oas.AnalyticsResultMetaCapsReached(c) + } + out.Meta.CapsReached = &caps + } + if next != "" { + out.NextCursor = &next + } + for i, r := range rows { + out.Rows[i] = oas.AnalyticsRow{Dimensions: slices.Clone(r.Dims), Metrics: apiMetricsBody(&r.Agg, r.Minutes, q.Metrics)} + } + if csv { + body := analyticsCSV(q.Dimensions, q.Metrics, out.Rows) + name := `attachment; filename="api-` + out.Meta.From.Format(time.DateOnly) + `-` + out.Meta.To.Format(time.DateOnly) + `.csv"` + return oas.QueryAnalytics200TextcsvResponse{Body: bytes.NewReader(body), ContentLength: int64(len(body)), + Headers: oas.QueryAnalytics200ResponseHeaders{ContentDisposition: &name}}, nil + } + return oas.QueryAnalytics200JSONResponse{Body: out}, nil +} diff --git a/api/internal/api/monitor_records.go b/api/internal/api/monitor_records.go new file mode 100644 index 00000000..55a0738e --- /dev/null +++ b/api/internal/api/monitor_records.go @@ -0,0 +1,410 @@ +package api + +import ( + "compress/gzip" + "context" + "errors" + "log/slog" + "net/http" + "net/netip" + "strings" + "time" + "uuid" + + "github.com/productdevbook/yuva/api/internal/analytics" + "github.com/productdevbook/yuva/api/internal/monitor" + "github.com/productdevbook/yuva/api/internal/oas" + "github.com/productdevbook/yuva/api/internal/store" +) + +const ( + maxRequestRecords = 1000 + maxRecordDurationUS = int64(24 * time.Hour / time.Microsecond) + maxSampleRate = 10_000 + defaultMonitorRecordsPerMinute = 600_000 + defaultMonitorBufferKeys = 200_000 + defaultMonitorMinuteDays = 8 + monitorBusyRetry = 10 +) + +var ( + errMonitorNeedsKey = problem(http.StatusForbidden, "forbidden", "only an API key can send request records") + errMonitorBusy = &apiError{Status: http.StatusServiceUnavailable, Code: "unavailable", Detail: "the server holds as many open rows as it may; try again shortly", + RetryAfter: monitorBusyRetry} + errEncoding = problem(http.StatusUnsupportedMediaType, "unsupported_media_type", "send the body as it is or with Content-Encoding: gzip") +) + +// MonitorSettings: zero values mean the defaults; Off removes API monitoring from the server. +type MonitorSettings struct { + Off bool + // MonthlyRequests caps each workspace's request records a month; 0 is no limit. + MonthlyRequests int64 + RecordsPerMinute int + BufferKeys int + MinuteDays int +} + +func monitorSettings(m MonitorSettings) MonitorSettings { + if m.RecordsPerMinute <= 0 { + m.RecordsPerMinute = defaultMonitorRecordsPerMinute + } + if m.BufferKeys <= 0 { + m.BufferKeys = defaultMonitorBufferKeys + } + if m.MinuteDays <= 0 { + m.MinuteDays = defaultMonitorMinuteDays + } + return m +} + +type monitorState struct { + settings MonitorSettings + store *monitor.PGStore + agg *monitor.Aggregator + usage *usageCache +} + +func (s *Server) newMonitor(d Deps) *monitorState { + set := monitorSettings(d.Analytics.Monitor) + st := &monitor.PGStore{Pool: d.Store.Pool} + return &monitorState{ + settings: set, store: st, usage: &usageCache{m: map[uuid.UUID]*usageEntry{}}, + agg: &monitor.Aggregator{Store: st, Log: s.log, Now: s.now, MaxKeys: set.BufferKeys}, + } +} + +func (s *Server) monitorOn() (*monitorState, error) { + if s.analytics == nil || s.analytics.monitor == nil { + return nil, errNotFound + } + return s.analytics.monitor, nil +} + +// FlushMonitor writes the open request rows now. +func (s *Server) FlushMonitor(ctx context.Context) error { + m, err := s.monitorOn() + if err != nil { + return nil + } + return m.agg.Flush(ctx) +} + +func (s *Server) workspaceMonitorLimit(override *int64) int64 { + if override != nil { + return *override + } + return s.analytics.monitor.settings.MonthlyRequests +} + +func (s *Server) loadMonitorUsage(ctx context.Context, ws uuid.UUID, now time.Time) (*usageEntry, error) { + month := monthOf(now) + c := s.analytics.monitor.usage + c.mu.Lock() + e, ok := c.m[ws] + c.mu.Unlock() + if ok && e.month.Equal(month) && now.Sub(e.loaded) < usageCacheTTL && !now.Before(e.loaded) { + return e, nil + } + u, err := s.st.GetMonitorUsage(ctx, store.GetMonitorUsageParams{WorkspaceID: ws, Month: month}) + if err != nil { + return nil, err + } + e = &usageEntry{month: month, limit: s.workspaceMonitorLimit(u.MonthlyLimit), events: u.Requests, loaded: now} + c.mu.Lock() + c.m[ws] = e + c.mu.Unlock() + return e, nil +} + +// takeMonitorUsage counts n records against the workspace's monthly cap; false means the cap is +// reached. +func (s *Server) takeMonitorUsage(ctx context.Context, ws uuid.UUID, n int, now time.Time) (bool, error) { + e, err := s.loadMonitorUsage(ctx, ws, now) + if err != nil { + return false, err + } + c := s.analytics.monitor.usage + c.mu.Lock() + defer c.mu.Unlock() + if e.limit > 0 && e.events+int64(n) > e.limit { + return false, nil + } + e.events += int64(n) + return true, nil +} + +func (s *Server) monitorUsage(ctx context.Context, ws uuid.UUID, out *oas.AnalyticsUsage) error { + if s.analytics.monitor == nil { + return nil + } + u, err := s.st.GetMonitorUsage(ctx, store.GetMonitorUsageParams{WorkspaceID: ws, Month: monthOf(s.now())}) + if err != nil { + return err + } + out.Requests = &u.Requests + if l := s.workspaceMonitorLimit(u.MonthlyLimit); l > 0 { + out.RequestsLimit = &l + } + return nil +} + +// OperatorLimitMonitor sets a workspace's monthly request records in place of +// YUVA_MONITOR_MONTHLY_REQUESTS; 0 is no limit. +func (s *Server) OperatorLimitMonitor(ctx context.Context, workspaceID uuid.UUID, requests int64) error { + if requests < 0 { + return errors.New("the limit must be a number of requests, 0 for no limit") + } + n, err := s.st.SetWorkspaceMonitorLimit(ctx, store.SetWorkspaceMonitorLimitParams{ID: workspaceID, MonthlyRequests: &requests}) + if err != nil { + return err + } + if n == 0 { + return errors.New("no such workspace") + } + return nil +} + +const requestsSuffix = "/requests" + +// gunzipRecords decompresses a request records body sent with Content-Encoding: gzip, up to the +// same size as any other body. +func (s *Server) gunzipRecords(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || !strings.HasPrefix(r.URL.Path, "/v1/analytics/properties/") || !strings.HasSuffix(r.URL.Path, requestsSuffix) { + next.ServeHTTP(w, r) + return + } + switch strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Encoding"))) { + case "", "identity": + case "gzip": + zr, err := gzip.NewReader(r.Body) + if err != nil { + writeProblem(w, errValidation("the body is not gzip")) + return + } + defer zr.Close() + r.Body, r.ContentLength = http.MaxBytesReader(w, zr, maxBodyBytes), -1 + default: + writeProblem(w, errEncoding) + return + } + next.ServeHTTP(w, r) + }) +} + +func (s *Server) SendAnalyticsRequests(ctx context.Context, req oas.SendAnalyticsRequestsRequestObject) (oas.SendAnalyticsRequestsResponseObject, error) { + m, err := s.monitorOn() + if err != nil { + return nil, err + } + p := principalFrom(ctx) + if !p.isKey() { + return nil, errMonitorNeedsKey + } + prop, err := visibleProperty(ctx, s.st.Queries, p, req.PropertyId, false) + if err != nil { + return nil, err + } + if !isAPIProperty(prop) { + return nil, errAPIOnly + } + if req.Body == nil || len(req.Body.Records) == 0 || len(req.Body.Records) > maxRequestRecords { + return nil, errValidation("records must hold 1 to 1,000 records") + } + now, loc := s.now(), propertyLocation(prop.Timezone) + day := monitor.LocalDay(now, loc) + count := func(reason string, n int) { + if n > 0 { + m.agg.Count(prop.WorkspaceID, prop.ID, day, reason, int64(n)) + } + } + rules := pathRules(prop.PathRules) + out := oas.AnalyticsRequestsResult{Errors: []oas.AnalyticsEventError{}} + recs := make([]monitor.Record, 0, len(req.Body.Records)) + late := 0 + for i := range req.Body.Records { + rec, isLate, err := s.requestRecord(&req.Body.Records[i], loc, rules, now) + if err != nil { + out.Errors = append(out.Errors, oas.AnalyticsEventError{Index: i, Detail: err.Error()}) + continue + } + out.Accepted++ + if isLate { + late++ + continue + } + recs = append(recs, rec) + } + count(monitor.ReasonInvalid, len(out.Errors)) + count(monitor.ReasonLate, late) + if len(recs) == 0 { + return oas.SendAnalyticsRequests202JSONResponse(out), nil + } + taken := 0 + for range recs { + if !s.analytics.limits.allow("m:"+prop.ID.String(), limit{m.settings.RecordsPerMinute, time.Minute}, now) || + !s.analytics.limits.allow("mk:"+p.keyID.String(), limit{m.settings.RecordsPerMinute, time.Minute}, now) { + break + } + taken++ + } + count(monitor.ReasonRate, len(recs)-taken) + recs = recs[:taken] + if len(recs) == 0 { + return oas.SendAnalyticsRequests202JSONResponse(out), nil + } + ok, err := s.takeMonitorUsage(ctx, prop.WorkspaceID, len(recs), now) + if err != nil { + return nil, err + } + if !ok { + count(monitor.ReasonOverLimit, len(recs)) + return oas.SendAnalyticsRequests202JSONResponse(out), nil + } + if err := m.agg.Add(ctx, monitor.Property{WorkspaceID: prop.WorkspaceID, ID: prop.ID, Location: loc}, recs); err != nil { + if errors.Is(err, monitor.ErrFull) { + count(monitor.ReasonBuffer, len(recs)) + return nil, errMonitorBusy + } + s.log.ErrorContext(ctx, "monitor records", slog.Any("error", err)) + return nil, err + } + count(monitor.ReasonAccepted, len(recs)) + return oas.SendAnalyticsRequests202JSONResponse(out), nil +} + +// recordCountry is the first and only use of a record's address: it is shortened, looked up for +// its country and dropped from the record. +func (s *Server) recordCountry(ip **string) string { + raw := derefString(*ip) + *ip = nil + if raw == "" { + return "" + } + addr, err := netip.ParseAddr(strings.TrimSpace(raw)) + if err != nil { + return "" + } + return s.analytics.geo.Lookup(analytics.ShortenAddr(addr)).Country +} + +func recordText(v *string, max int, name string) (string, error) { + if v == nil { + return "", nil + } + if len(*v) > max { + return "", errors.New(name + " has at most " + formatCount(int64(max)) + " characters") + } + return analytics.Clean(*v, max), nil +} + +// requestRecord checks one request record; late reports a valid record older than 72 hours. +func (s *Server) requestRecord(r *oas.AnalyticsRequestRecord, loc *time.Location, rules []analytics.PathRule, now time.Time) (monitor.Record, bool, error) { + country := s.recordCountry(&r.Ip) + out := monitor.Record{Country: country, Source: monitor.SourceServer, Platform: monitor.PlatformNone, RequestBytes: -1, ResponseBytes: -1, Weight: 1} + if m := strings.TrimSpace(r.Method); m == "" || len(m) > 20 { + return out, false, errors.New("method must be 1 to 20 characters") + } + out.Method = monitor.Method(r.Method) + if r.Status != 0 && (r.Status < 100 || r.Status > 599) { + return out, false, errors.New("status must be 100 to 599, or 0 when no response was sent") + } + out.Status = int16(r.Status) + if r.DurationUs < 0 || r.DurationUs > maxRecordDurationUS { + return out, false, errors.New("duration_us must be 0 to 86,400,000,000 microseconds") + } + out.DurationUS = r.DurationUs + for _, size := range []struct { + v *int64 + dst *int64 + name string + }{{r.RequestSize, &out.RequestBytes, "request_size"}, {r.ResponseSize, &out.ResponseBytes, "response_size"}} { + if size.v == nil { + continue + } + if *size.v < 0 { + return out, false, errors.New(size.name + " must not be negative") + } + *size.dst = *size.v + } + route, inferred, err := recordRoute(r, out.Status, rules) + if err != nil { + return out, false, err + } + out.Route, out.Inferred = route, inferred + if r.SampleRate != nil { + if *r.SampleRate < 1 || *r.SampleRate > maxSampleRate { + return out, false, errors.New("sample_rate must be 1 to 10,000") + } + if *r.SampleRate > 1 && (out.Status == 0 || out.Status >= 400) { + return out, false, errors.New("only records with a status of 1 to 399 may be sampled") + } + out.Weight = int64(*r.SampleRate) + } + if r.Consumer != nil { + c := analytics.Clean(*r.Consumer, 100) + if c == "" || len(*r.Consumer) > 100 { + return out, false, errors.New("consumer must be 1 to 100 characters") + } + out.Consumer = monitor.ScrubConsumer(c) + } + if out.APIVersion, err = recordText(r.ApiVersion, 40, "api_version"); err != nil { + return out, false, err + } + if out.Environment, err = recordText(r.Environment, 40, "environment"); err != nil { + return out, false, err + } + if out.Environment == "" { + out.Environment = monitor.DefaultEnvironment + } + if out.Service, err = recordText(r.Service, 100, "service"); err != nil { + return out, false, err + } + if _, err = recordText(r.Release, 80, "release"); err != nil { + return out, false, err + } + var at time.Time + if r.Time != nil { + at = *r.Time + } + at, late := eventTime(at, now) + out.Time, out.Hour, out.Day = at, monitor.LocalHour(at, loc), monitor.LocalDay(at, loc) + return out, late, nil +} + +// recordRoute is the route a record is counted under: the property's path rules first, then the +// record's pattern with any id-shaped segment taken out, or a pattern inferred from its path. A +// path that answered 404 or 405 is (unmatched). +func recordRoute(r *oas.AnalyticsRequestRecord, status int16, rules []analytics.PathRule) (string, bool, error) { + var raw string + inferred := false + switch { + case r.Route != nil: + raw = strings.TrimSpace(*r.Route) + if !strings.HasPrefix(raw, "/") || len(raw) > monitor.MaxRoute { + return "", false, errors.New("route must be a pattern of at most 300 characters starting with /") + } + case r.Path != nil: + raw = strings.TrimSpace(*r.Path) + if len(raw) > analytics.MaxURLLength { + return "", false, errors.New("path has at most 2,000 characters") + } + if i := strings.IndexAny(raw, "?#"); i >= 0 { + raw = raw[:i] + } + if status == http.StatusNotFound || status == http.StatusMethodNotAllowed { + return monitor.Unmatched, false, nil + } + inferred = true + default: + return "", false, errors.New("give route, or path when the router knows no pattern") + } + if mapped := analytics.ApplyRules(rules, raw); mapped != raw { + return mapped, false, nil + } + route := monitor.InferRoute(raw) + if len(route) > monitor.MaxRoute { + return "", false, errors.New("the route has more than 300 characters") + } + return route, inferred, nil +} diff --git a/api/internal/api/monitor_test.go b/api/internal/api/monitor_test.go new file mode 100644 index 00000000..8752a5ed --- /dev/null +++ b/api/internal/api/monitor_test.go @@ -0,0 +1,463 @@ +package api_test + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "math" + "math/rand/v2" + "net/http" + "slices" + "strings" + "testing" + "time" + "uuid" + + "github.com/productdevbook/yuva/api/internal/api" +) + +func sendRecords(c *client, want int, id string, recs ...map[string]any) response { + c.h.t.Helper() + return c.expect(want, "POST", "/v1/analytics/properties/"+id+"/requests", map[string]any{"records": recs}) +} + +func apiQuery(c *client, id string, q map[string]any) response { + c.h.t.Helper() + q["dataset"] = "api" + return query(c, id, q) +} + +func createAPIProperty(c *client, body map[string]any) string { + c.h.t.Helper() + body["kind"] = "api" + if _, ok := body["domains"]; !ok { + body["domains"] = []string{} + } + id, _ := createProperty(c, body) + return id +} + +func (h *harness) flushMonitor() { + h.t.Helper() + if err := h.srv.FlushMonitor(context.Background()); err != nil { + h.t.Fatal(err) + } +} + +// sent is what the test sent of one route, to check the answers against. +type sent struct { + requests, errors, clientErrors int + durations []int64 +} + +func (s *sent) percentile(q float64) float64 { + d := slices.Clone(s.durations) + slices.Sort(d) + rank := min(max(int(math.Ceil(q*float64(len(d)))), 1), len(d)) + return float64(d[rank-1]) / 1000 +} + +func metricOf(m map[string]any, key string) float64 { + v, _ := m[key].(float64) + return v +} + +func within(got, want, rel float64) bool { + if want == 0 { + return got == 0 + } + return math.Abs(got-want)/want <= rel +} + +var apiMetrics = []string{"requests", "errors", "error_rate", "client_errors", "client_error_rate", "p50", "p95", "p99", "avg_duration"} + +func TestMonitorCountsRoutesAndPercentiles(t *testing.T) { + h := geoHarness(t, nil) + tm := newTeam(t, h) + id := createAPIProperty(tm.owner, map[string]any{"name": "Shop API", "domains": []string{"api.example.com"}, "timezone": "Asia/Kolkata"}) + ctx := context.Background() + now := time.Date(2026, 10, 12, 9, 0, 30, 0, time.UTC) + h.clock.Set(now) + backend := tm.apiKey(h) + + r := rand.New(rand.NewPCG(11, 12)) + want := map[string]*sent{} + var batch []map[string]any + add := func(route string, rec map[string]any) { + s := want[route] + if s == nil { + s = &sent{} + want[route] = s + } + status := rec["status"].(int) + s.requests++ + if status == 0 || status >= 500 { + s.errors++ + } else if status >= 400 { + s.clientErrors++ + } + s.durations = append(s.durations, rec["duration_us"].(int64)) + batch = append(batch, rec) + } + for i := range 2400 { + dur := int64(math.Exp(r.NormFloat64()*0.9+9.5)) + 1 + at := now.Add(-time.Duration(r.IntN(150)) * time.Second).Format(time.RFC3339Nano) + switch i % 6 { + case 0, 1, 2: + status := []int{200, 200, 200, 201, 404, 500, 503, 0}[r.IntN(8)] + add("/users/{id}", map[string]any{"method": "GET", "route": "/users/{id}", "status": status, "duration_us": dur, "time": at, + "consumer": []string{"mobile-app", "partner-api", "ayse@example.com"}[i%3], "ip": []string{"203.0.113.7", "198.51.100.20", "2001:db8:1::7"}[i%3], + "response_size": int64(200 + r.IntN(5000))}) + case 3: + add("/orders/{id}/items", map[string]any{"method": "post", "path": fmt.Sprintf("/orders/%d/items?coupon=SPRING", 1000+r.IntN(900)), "status": 201, + "duration_us": dur * 3, "time": at, "environment": "staging"}) + case 4: + add("/files/{id}", map[string]any{"method": "GET", "path": "/files/" + uuid.New().String(), "status": []int{200, 304, 403}[r.IntN(3)], + "duration_us": dur / 2, "time": at}) + case 5: + add("(unmatched)", map[string]any{"method": "GET", "path": fmt.Sprintf("/wp-admin/%d/setup.php", r.IntN(1e6)), "status": 404, "duration_us": int64(80 + r.IntN(40)), + "time": at}) + } + } + for start := 0; start < len(batch); start += 1000 { + res := sendRecords(backend, http.StatusAccepted, id, batch[start:min(start+1000, len(batch))]...) + if len(res.body["errors"].([]any)) != 0 { + t.Fatalf("records refused: %s", res.raw) + } + } + invalid := sendRecords(backend, http.StatusAccepted, id, + map[string]any{"method": "GET", "route": "/ok", "status": 200, "duration_us": 10}, + map[string]any{"method": "GET", "status": 200, "duration_us": 10}, + map[string]any{"method": "GET", "route": "/x", "status": 700, "duration_us": 10}, + map[string]any{"method": "GET", "route": "/x", "status": 500, "duration_us": 10, "sample_rate": 10}, + map[string]any{"method": "GET", "route": "/old", "status": 200, "duration_us": 10, "time": now.Add(-73 * time.Hour).Format(time.RFC3339)}, + ) + if invalid.body["accepted"].(float64) != 2 || len(invalid.body["errors"].([]any)) != 3 { + t.Fatalf("invalid records %s", invalid.raw) + } + add("/ok", map[string]any{"method": "GET", "status": 200, "duration_us": int64(10)}) + h.flushMonitor() + + if n := h.count("SELECT count(*) FROM monitor_minute WHERE workspace_id = $1 AND property_id = $2", tm.ws, id); n == 0 || n > 300 { + t.Fatalf("%d minute rows for %d records", n, len(batch)) + } + for _, table := range []string{"monitor_minute", "monitor_dim_hour", "monitor_names", "monitor_routes", "monitor_rollup_hours", "monitor_ingest_stats"} { + var dump string + if err := h.st.Pool.QueryRow(ctx, "SELECT coalesce(string_agg(t::text, '|'), '') FROM "+table+" t WHERE t.workspace_id = $1", tm.ws).Scan(&dump); err != nil { + t.Fatal(err) + } + for _, addr := range []string{"203.0.113", "198.51.100", "2001:db8", "ayse@example.com", "SPRING"} { + if strings.Contains(dump, addr) { + t.Fatalf("%s holds %q", table, addr) + } + } + } + var routes []string + rows, err := h.st.Pool.Query(ctx, "SELECT pattern FROM monitor_routes WHERE workspace_id = $1 AND property_id = $2 AND pattern <> '(other)' ORDER BY pattern", tm.ws, id) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var p string + _ = rows.Scan(&p) + routes = append(routes, p) + } + rows.Close() + if fmt.Sprint(routes) != "[(unmatched) /files/{id} /ok /orders/{id}/items /users/{id}]" { + t.Fatalf("routes %v", routes) + } + var countries []string + if err := h.st.Pool.QueryRow(ctx, "SELECT array_agg(value ORDER BY value) FROM monitor_names WHERE workspace_id = $1 AND kind = 'country' AND value <> '(other)'", tm.ws).Scan(&countries); err != nil { + t.Fatal(err) + } + if fmt.Sprint(countries) != "[DE TR US]" { + t.Fatalf("countries %v", countries) + } + if n := h.count("SELECT count(*) FROM monitor_names WHERE workspace_id = $1 AND kind = 'consumer' AND value = '(scrubbed)'", tm.ws); n != 1 { + t.Fatal("the e-mail-shaped consumer is not scrubbed") + } + + check := func(label string, res response) { + t.Helper() + got := rowsBy(res) + total := 0 + for route, s := range want { + m := got[route] + if m == nil { + t.Fatalf("%s: no row for %s: %s", label, route, res.raw) + } + total += s.requests + errRate := math.Round(float64(s.errors)*10000/float64(s.requests)) / 100 + if m["requests"].(float64) != float64(s.requests) || m["errors"].(float64) != float64(s.errors) || + m["client_errors"].(float64) != float64(s.clientErrors) || m["error_rate"].(float64) != errRate { + t.Fatalf("%s: %s counts %v, sent %+v", label, route, m, *s) + } + for _, p := range []struct { + name string + q float64 + }{{"p50", 0.5}, {"p95", 0.95}, {"p99", 0.99}} { + if !within(m[p.name].(float64), s.percentile(p.q), 0.011) { + t.Fatalf("%s: %s %s %v, exact %v", label, route, p.name, m[p.name], s.percentile(p.q)) + } + } + } + if len(got) != len(want) || totals(res)["requests"].(float64) != float64(total) { + t.Fatalf("%s: rows %v", label, got) + } + } + day := "2026-10-12" + byRoute := map[string]any{"metrics": apiMetrics, "range": dayRange(day), "dimensions": []string{"route"}} + live := apiQuery(tm.owner, id, byRoute) + check("minute rows", live) + if served(live) != "raw" { + t.Fatalf("served from %s", served(live)) + } + minutes := apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests", "errors"}, "range": map[string]any{"period": "15m"}, "dimensions": []string{"time:minute"}}) + list := minutes.body["rows"].([]any) + sum := 0.0 + for _, row := range list { + sum += row.(map[string]any)["metrics"].(map[string]any)["requests"].(float64) + } + if len(list) != 15 || sum != totals(live)["requests"].(float64) || !strings.HasSuffix(list[len(list)-1].(map[string]any)["dimensions"].([]any)[0].(string), "T14:30") { + t.Fatalf("minutes %s", minutes.raw) + } + statuses := rowsBy(apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests"}, "range": dayRange(day), "dimensions": []string{"status_class"}, + "filters": []map[string]any{{"dimension": "route", "op": "is", "values": []string{"/users/{id}"}}}})) + u := want["/users/{id}"] + if metricOf(statuses["5xx"], "requests")+metricOf(statuses["none"], "requests") != float64(u.errors) || metricOf(statuses["4xx"], "requests") != float64(u.clientErrors) { + t.Fatalf("status classes %v", statuses) + } + rate := totals(apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests", "requests_per_minute"}, "range": map[string]any{"period": "1h"}})) + if rate["requests_per_minute"].(float64) != math.Round(rate["requests"].(float64)/60*100)/100 { + t.Fatalf("requests per minute %v", rate) + } + staging := totals(apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests"}, "range": dayRange(day), + "filters": []map[string]any{{"dimension": "environment", "op": "is", "values": []string{"staging"}}}})) + if staging["requests"].(float64) != float64(want["/orders/{id}/items"].requests) { + t.Fatalf("staging %v", staging) + } + + h.clock.Set(time.Date(2026, 10, 13, 6, 0, 0, 0, time.UTC)) + h.flushMonitor() + if err := h.srv.RollupMonitor(ctx, uuid.MustParse(tm.ws), uuid.MustParse(id)); err != nil { + t.Fatal(err) + } + if n := h.count("SELECT count(*) FROM monitor_rollup_hours WHERE workspace_id = $1 AND property_id = $2 AND (built_at IS NULL OR day_built_at IS NULL)", tm.ws, id); n != 0 { + t.Fatalf("%d hours or days not folded", n) + } + folded := apiQuery(tm.owner, id, byRoute) + check("folded days", folded) + if served(folded) != "rollups" || numbers(folded) != numbers(live) { + t.Fatalf("folded days differ from minute rows:\nminutes %s\ndays %s", numbers(live), numbers(folded)) + } + hours := apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests"}, "range": dayRange(day), "dimensions": []string{"time:hour"}}) + if hl := hours.body["rows"].([]any); len(hl) != 24 || hl[14].(map[string]any)["dimensions"].([]any)[0] != "2026-10-12T14:00" { + t.Fatalf("local hours %s", hours.raw) + } +} + +func TestMonitorWritersLateRecordsAndRefolds(t *testing.T) { + h := newHarness(t) + tm := newTeam(t, h) + id := createAPIProperty(tm.owner, map[string]any{"name": "API"}) + ctx := context.Background() + now := time.Date(2026, 10, 12, 12, 0, 20, 0, time.UTC) + h.clock.Set(now) + backend := tm.apiKey(h) + rec := func(at time.Time, status int) map[string]any { + return map[string]any{"method": "GET", "route": "/ping", "status": status, "duration_us": 1500, "time": at.Format(time.RFC3339)} + } + sendRecords(backend, http.StatusAccepted, id, rec(now, 200), rec(now, 500)) + h.flushMonitor() + h.flushMonitor() + sendRecords(backend, http.StatusAccepted, id, rec(now, 200)) + h.flushMonitor() + sum := func() int { + return h.count("SELECT coalesce(sum(requests), 0)::int FROM monitor_minute WHERE workspace_id = $1 AND property_id = $2", tm.ws, id) + } + if n := h.count("SELECT count(*) FROM monitor_minute WHERE workspace_id = $1 AND property_id = $2 AND status = 200", tm.ws, id); n != 1 || sum() != 3 { + t.Fatalf("cumulative upserts: %d rows, %d requests", n, sum()) + } + + h.clock.Set(time.Date(2026, 10, 12, 13, 10, 0, 0, time.UTC)) + h.flushMonitor() + if err := h.srv.RollupMonitor(ctx, uuid.MustParse(tm.ws), uuid.MustParse(id)); err != nil { + t.Fatal(err) + } + if n := h.count("SELECT count(*) FROM monitor_hour WHERE workspace_id = $1 AND property_id = $2", tm.ws, id); n != 2 { + t.Fatalf("%d hour rows", n) + } + sendRecords(backend, http.StatusAccepted, id, rec(now.Add(5*time.Minute), 200)) + h.flushMonitor() + if n := h.count("SELECT count(DISTINCT writer) FROM monitor_minute WHERE workspace_id = $1 AND property_id = $2", tm.ws, id); n != 2 || sum() != 4 { + t.Fatalf("a late record is written under a fresh writer: %d writers, %d requests", n, sum()) + } + if n := h.count("SELECT count(*) FROM monitor_rollup_hours WHERE workspace_id = $1 AND property_id = $2 AND dirty", tm.ws, id); n != 1 { + t.Fatal("the late record did not mark its hour dirty") + } + if err := h.srv.RollupMonitor(ctx, uuid.MustParse(tm.ws), uuid.MustParse(id)); err != nil { + t.Fatal(err) + } + if n := h.count("SELECT sum(requests)::int FROM monitor_hour WHERE workspace_id = $1 AND property_id = $2", tm.ws, id); n != 4 { + t.Fatalf("refolded hour holds %d requests", n) + } + q := apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests", "errors"}, "range": dayRange("2026-10-12")}) + if totals(q)["requests"].(float64) != 4 || totals(q)["errors"].(float64) != 1 || served(q) != "rollups" { + t.Fatalf("after the refold %s", q.raw) + } +} + +func TestMonitorAccessAndKinds(t *testing.T) { + h := newHarness(t) + tm := newTeam(t, h) + other := newTeam(t, h) + id := createAPIProperty(tm.owner, map[string]any{"name": "API"}) + site, siteKey := createProperty(tm.owner, map[string]any{"name": "Site", "domains": []string{"example.com"}}) + rec := map[string]any{"method": "GET", "route": "/ping", "status": 200, "duration_us": 900} + body := map[string]any{"records": []any{rec}} + path := "/v1/analytics/properties/" + id + "/requests" + + reader := h.client() + reader.bearer = tm.owner.expect(http.StatusCreated, "POST", "/v1/api-keys", map[string]any{"name": "reader", "scopes": []string{"analytics:read"}}).str("secret") + reader.expectProblem(http.StatusForbidden, "insufficient_scope", "POST", path, body) + tm.owner.expectProblem(http.StatusForbidden, "forbidden", "POST", path, body) + other.apiKey(h).expectProblem(http.StatusNotFound, "not_found", "POST", path, body) + other.owner.expectProblem(http.StatusNotFound, "not_found", "POST", "/v1/analytics/properties/"+id+"/query", map[string]any{"dataset": "api", "metrics": []string{"requests"}, "range": map[string]any{"period": "1h"}}) + backend := tm.apiKey(h) + backend.expectProblem(http.StatusBadRequest, "validation_failed", "POST", path, map[string]any{"records": []any{}}) + backend.expectProblem(http.StatusUnprocessableEntity, "property_kind", "POST", "/v1/analytics/properties/"+site+"/requests", body) + sendRecords(backend, http.StatusAccepted, id, rec) + + tm.owner.expectProblem(http.StatusUnprocessableEntity, "property_kind", "POST", "/v1/analytics/properties/"+site+"/query", + map[string]any{"dataset": "api", "metrics": []string{"requests"}, "range": map[string]any{"period": "1h"}}) + tm.owner.expectProblem(http.StatusUnprocessableEntity, "property_kind", "POST", "/v1/analytics/properties/"+id+"/query", + map[string]any{"metrics": []string{"visitors"}, "range": map[string]any{"period": "today"}}) + tm.owner.expectProblem(http.StatusBadRequest, "validation_failed", "POST", "/v1/analytics/properties/"+site+"/query", + map[string]any{"metrics": []string{"visitors"}, "range": map[string]any{"period": "1h"}}) + tm.owner.expectProblem(http.StatusBadRequest, "validation_failed", "POST", "/v1/analytics/properties/"+id+"/query", + map[string]any{"dataset": "api", "metrics": []string{"visitors"}, "range": map[string]any{"period": "1h"}}) + tm.owner.expectProblem(http.StatusUnprocessableEntity, "range_too_large", "POST", "/v1/analytics/properties/"+id+"/query", + map[string]any{"dataset": "api", "metrics": []string{"requests"}, "range": map[string]any{"period": "7d"}, "dimensions": []string{"time:minute"}}) + backend.expectProblem(http.StatusUnprocessableEntity, "property_kind", "POST", "/v1/analytics/properties/"+id+"/events", + map[string]any{"events": []any{map[string]any{"name": "pageview", "url": "https://example.com/", "visitor_key": "k"}}}) + tm.owner.expectProblem(http.StatusUnprocessableEntity, "property_kind", "POST", "/v1/analytics/properties/"+id+"/goals", + map[string]any{"name": "Signed up", "kind": "event", "event_name": "Signup"}) + tm.owner.expectProblem(http.StatusBadRequest, "validation_failed", "POST", "/v1/analytics/properties", + map[string]any{"kind": "api", "name": "Web", "domains": []string{}, "platforms": []string{"web"}}) + tm.owner.expectProblem(http.StatusBadRequest, "validation_failed", "POST", "/v1/analytics/properties", + map[string]any{"name": "No domains", "domains": []string{}}) + got := tm.owner.expect(http.StatusOK, "GET", "/v1/analytics/properties/"+id, nil) + if got.str("kind") != "api" || got.body["apdex_ms"].(float64) != 500 || got.body["monitor_hour_days"].(float64) != 90 || len(got.body["platforms"].([]any)) != 0 { + t.Fatalf("API property %s", got.raw) + } + tm.owner.expect(http.StatusOK, "PATCH", "/v1/analytics/properties/"+id, map[string]any{"apdex_ms": 250, "monitor_hour_days": 30, "platforms": []string{"ios"}}) + tm.owner.expectProblem(http.StatusBadRequest, "validation_failed", "PATCH", "/v1/analytics/properties/"+id, map[string]any{"monitor_hour_days": 3}) + var pubKey string + if err := h.st.Pool.QueryRow(context.Background(), "SELECT public_key FROM analytics_properties WHERE id = $1", id).Scan(&pubKey); err != nil { + t.Fatal(err) + } + if r := collect(h, pubKey, collectReq{origin: "https://example.com", ip: "203.0.113.7", body: pageview("https://example.com/")}); r.status != http.StatusNotFound { + t.Fatalf("collect with an API property's key: %d", r.status) + } + if r := collect(h, siteKey, collectReq{origin: "https://example.com", ip: "203.0.113.7", body: pageview("https://example.com/")}); r.status != http.StatusAccepted { + t.Fatalf("collect on the site: %d", r.status) + } + + var zipped bytes.Buffer + zw := gzip.NewWriter(&zipped) + _ = json.NewEncoder(zw).Encode(map[string]any{"records": []any{rec, rec}}) + _ = zw.Close() + req, _ := http.NewRequest(http.MethodPost, h.url+path, &zipped) + req.Header.Set("Authorization", "Bearer "+backend.bearer) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Encoding", "gzip") + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + var out map[string]any + _ = json.NewDecoder(res.Body).Decode(&out) + res.Body.Close() + if res.StatusCode != http.StatusAccepted || out["accepted"].(float64) != 2 { + t.Fatalf("gzip body: %d %v", res.StatusCode, out) + } + req, _ = http.NewRequest(http.MethodPost, h.url+path, strings.NewReader("x")) + req.Header.Set("Authorization", "Bearer "+backend.bearer) + req.Header.Set("Content-Encoding", "br") + if res, err = http.DefaultClient.Do(req); err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("brotli body: %d", res.StatusCode) + } +} + +func TestMonitorUsageLimitAndCaps(t *testing.T) { + h := newHarness(t) + tm := newTeam(t, h) + id := createAPIProperty(tm.owner, map[string]any{"name": "API"}) + h.clock.Set(time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)) + backend := tm.apiKey(h) + if err := h.srv.OperatorLimitMonitor(context.Background(), uuid.MustParse(tm.ws), 1500); err != nil { + t.Fatal(err) + } + var recs []map[string]any + for i := range 1000 { + recs = append(recs, map[string]any{"method": "GET", "route": fmt.Sprintf("/r%d", i), "status": 200, "duration_us": 100}) + } + sendRecords(backend, http.StatusAccepted, id, recs...) + sendRecords(backend, http.StatusAccepted, id, map[string]any{"method": "GET", "route": "/one-too-many", "status": 200, "duration_us": 100}) + sendRecords(backend, http.StatusAccepted, id, recs[:600]...) + h.flushMonitor() + usage := tm.owner.expect(http.StatusOK, "GET", "/v1/analytics/properties", nil).body["usage"].(map[string]any) + if usage["requests"].(float64) != 1001 || usage["requests_limit"].(float64) != 1500 { + t.Fatalf("usage %v", usage) + } + if n := h.count("SELECT records::int FROM monitor_ingest_stats WHERE workspace_id = $1 AND property_id = $2 AND reason = 'over_limit'", tm.ws, id); n != 600 { + t.Fatalf("%d records over the limit", n) + } + res := apiQuery(tm.owner, id, map[string]any{"metrics": []string{"requests"}, "range": dayRange("2026-10-12"), "dimensions": []string{"route"}, "limit": 1000}) + byRoute := rowsBy(res) + caps := res.body["meta"].(map[string]any)["caps_reached"].([]any) + if byRoute["(other)"]["requests"].(float64) != 1 || byRoute["/one-too-many"] != nil || len(caps) != 1 || caps[0] != "routes" { + t.Fatalf("route cap: other %v, caps %v", byRoute["(other)"], caps) + } + if res.body["next_cursor"] == nil { + t.Fatal("1,001 routes fit on one page of 1,000") + } +} + +func TestMonitorOff(t *testing.T) { + h := newHarnessWith(t, func(d *api.Deps) { d.Analytics.Monitor.Off = true }) + tm := newTeam(t, h) + site, _ := createProperty(tm.owner, map[string]any{"name": "Site", "domains": []string{"example.com"}}) + tm.owner.expectProblem(http.StatusBadRequest, "validation_failed", "POST", "/v1/analytics/properties", map[string]any{"kind": "api", "name": "API", "domains": []string{}}) + tm.apiKey(h).expectProblem(http.StatusNotFound, "not_found", "POST", "/v1/analytics/properties/"+site+"/requests", + map[string]any{"records": []any{map[string]any{"method": "GET", "route": "/", "status": 200, "duration_us": 1}}}) + if u := tm.owner.expect(http.StatusOK, "GET", "/v1/analytics/properties", nil).body["usage"].(map[string]any); u["requests"] != nil { + t.Fatalf("usage %v", u) + } +} + +func TestMonitorDeleteProperty(t *testing.T) { + h := newHarness(t) + tm := newTeam(t, h) + id := createAPIProperty(tm.owner, map[string]any{"name": "API"}) + h.clock.Set(time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)) + sendRecords(tm.apiKey(h), http.StatusAccepted, id, map[string]any{"method": "GET", "route": "/", "status": 200, "duration_us": 1, "consumer": "c"}) + h.flushMonitor() + ws, prop := uuid.MustParse(tm.ws), uuid.MustParse(id) + tm.owner.expect(http.StatusAccepted, "DELETE", "/v1/analytics/properties/"+id, map[string]any{"name": "API"}) + done, err := h.srv.DeleteAnalytics(context.Background(), api.AnalyticsDeleteArgs{WorkspaceID: ws, PropertyID: prop, Property: true}, time.Minute) + if err != nil || !done { + t.Fatalf("delete %v %v", done, err) + } + for _, table := range []string{"monitor_minute", "monitor_dim_hour", "monitor_rollup_hours", "monitor_routes", "monitor_names"} { + if n := h.count("SELECT count(*) FROM "+table+" WHERE workspace_id = $1 AND property_id = $2", tm.ws, id); n != 0 { + t.Fatalf("%s keeps %d rows", table, n) + } + } +} diff --git a/api/internal/api/scopes.go b/api/internal/api/scopes.go index f080b424..3f312cf3 100644 --- a/api/internal/api/scopes.go +++ b/api/internal/api/scopes.go @@ -218,6 +218,7 @@ var operationScopes = map[string][]oas.ApiKeyScope{ "GetAnalyticsRetention": {oas.AnalyticsRead}, "DeleteAnalyticsVisitor": {oas.AnalyticsManage}, "SendAnalyticsEvents": {oas.AnalyticsWrite}, + "SendAnalyticsRequests": {oas.AnalyticsWrite}, "ListAnalyticsGoals": {oas.AnalyticsRead}, "CreateAnalyticsGoal": {oas.AnalyticsManage}, diff --git a/api/internal/api/server.go b/api/internal/api/server.go index 9fe6e1f3..7b45316e 100644 --- a/api/internal/api/server.go +++ b/api/internal/api/server.go @@ -375,7 +375,7 @@ func (s *Server) Handler() http.Handler { } panel.ServeHTTP(w, r) }) - return s.recoverer(s.logRequests(noSniff(s.linkHost(oauthCORS(s.clientCORS(s.guardCookieWrites(s.limitBody(s.idempotency(mux))))))))) + return s.recoverer(s.logRequests(noSniff(s.linkHost(oauthCORS(s.clientCORS(s.guardCookieWrites(s.limitBody(s.gunzipRecords(s.idempotency(mux)))))))))) } var mailLinkPrefixes = []string{managePathPrefix, confirmPathPrefix, clickPathPrefix, emailImagePathPrefix} diff --git a/api/internal/config/config.go b/api/internal/config/config.go index 561f1b2b..2dafdf26 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -127,6 +127,18 @@ type Analytics struct { GeoIP string GeoIPDir string GeoIPFile string + Monitor Monitor +} + +// Monitor is API monitoring (YUVA_MONITOR), part of analytics: off with either. +type Monitor struct { + On bool + // MonthlyRequests caps each workspace's request records a month; 0 is no limit. A server with + // open sign-up defaults to 10,000,000. + MonthlyRequests int64 + RecordsPerMinute int + BufferKeys int + MinuteDays int } type VAPID struct { @@ -389,6 +401,9 @@ func (c *Config) loadAnalytics() error { if a.RollupTop, err = strconv.Atoi(env("YUVA_ANALYTICS_ROLLUP_TOP", "1000")); err != nil || a.RollupTop < 10 || a.RollupTop > 100000 { return errors.New("YUVA_ANALYTICS_ROLLUP_TOP must be 10 to 100000 values") } + if err := c.loadMonitor(); err != nil { + return err + } a.GeoIP = strings.ToLower(env("YUVA_GEOIP", "dbip")) a.GeoIPDir = env("YUVA_GEOIP_DIR", "data/geoip") a.GeoIPFile = env("YUVA_GEOIP_FILE", "") @@ -404,6 +419,31 @@ func (c *Config) loadAnalytics() error { return nil } +func (c *Config) loadMonitor() error { + m := &c.Analytics.Monitor + var err error + if m.On, err = switchEnv("YUVA_MONITOR", true); err != nil { + return err + } + def := "0" + if c.SignupOpen { + def = "10000000" + } + if m.MonthlyRequests, err = strconv.ParseInt(env("YUVA_MONITOR_MONTHLY_REQUESTS", def), 10, 64); err != nil || m.MonthlyRequests < 0 { + return errors.New("YUVA_MONITOR_MONTHLY_REQUESTS must be a number of requests, 0 for no limit") + } + if m.RecordsPerMinute, err = strconv.Atoi(env("YUVA_MONITOR_RECORDS_PER_MINUTE", "600000")); err != nil || m.RecordsPerMinute < 1 { + return errors.New("YUVA_MONITOR_RECORDS_PER_MINUTE must be a positive number of records") + } + if m.BufferKeys, err = strconv.Atoi(env("YUVA_MONITOR_BUFFER_KEYS", "200000")); err != nil || m.BufferKeys < 1000 { + return errors.New("YUVA_MONITOR_BUFFER_KEYS must be a number of rows, at least 1000") + } + if m.MinuteDays, err = strconv.Atoi(env("YUVA_MONITOR_MINUTE_DAYS", "8")); err != nil || m.MinuteDays < 2 || m.MinuteDays > 31 { + return errors.New("YUVA_MONITOR_MINUTE_DAYS must be 2 to 31 days") + } + return nil +} + var migrationSources = []string{"imap", "file", "zendesk", "helpscout"} var forwardingDomainRe = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$`) diff --git a/api/internal/monitor/aggregate.go b/api/internal/monitor/aggregate.go new file mode 100644 index 00000000..253c6b90 --- /dev/null +++ b/api/internal/monitor/aggregate.go @@ -0,0 +1,615 @@ +package monitor + +import ( + "cmp" + "context" + "errors" + "log/slog" + "maps" + "math/rand/v2" + "slices" + "sync" + "time" + "uuid" +) + +const ( + FlushInterval = 10 * time.Second + // LastFlush is how long after its minute or hour ends a bucket is written a last time and + // leaves memory; records for it after that are late. + LastFlush = 2 * time.Minute + // FoldAfter is how long after it ends an hour is folded. It outlasts every process's last + // flush of the hour, so no writer still replaces its rows once they are compacted. + FoldAfter = 5 * time.Minute + drainTimeout = 10 * time.Second + maxBackoff = time.Minute + namesTTL = 10 * time.Minute + maxNames = 2_000_000 +) + +// ErrFull: the aggregator holds as many open rows as it may. +var ErrFull = errors.New("monitor: too many open rows") + +type coreCellKey struct { + ws, prop uuid.UUID + minute time.Time + key CoreKey +} + +type dimCellKey struct { + ws, prop uuid.UUID + hour time.Time + key DimKey +} + +type hourKey struct { + prop uuid.UUID + hour time.Time +} + +type cell struct { + hour, day time.Time + agg Agg + dirty bool + flushed bool + final time.Time +} + +type countKey struct { + ws, prop uuid.UUID + day time.Time + reason string +} + +// Aggregator adds records up in memory, per UTC minute for the core key and per local hour for +// the extra dimensions, and writes each bucket's cumulative totals under its own writer id every +// ten seconds, replacing what it wrote before. A bucket is written a last time LastFlush after it +// ends and then forgotten; records for a bucket already gone are written once under a fresh +// writer. +type Aggregator struct { + Store Store + Log *slog.Logger + Now func() time.Time + MaxKeys int + + mu sync.Mutex + writer int64 + core map[coreCellKey]*cell + dims map[dimCellKey]*cell + lateCore map[coreCellKey]*cell + lateDims map[dimCellKey]*cell + dimRows map[hourKey]int + capped map[hourKey]bool + counts map[countKey]int64 + usage map[uuid.UUID]int64 + + names nameCache + partMu sync.Mutex + minuteDay map[time.Time]bool + hourMonth map[time.Time]bool +} + +func (a *Aggregator) init() { + if a.core == nil { + a.writer = newWriter() + a.core, a.dims = map[coreCellKey]*cell{}, map[dimCellKey]*cell{} + a.lateCore, a.lateDims = map[coreCellKey]*cell{}, map[dimCellKey]*cell{} + a.dimRows, a.capped = map[hourKey]int{}, map[hourKey]bool{} + a.counts, a.usage = map[countKey]int64{}, map[uuid.UUID]int64{} + } +} + +func newWriter() int64 { return rand.Int64N(1<<62) + 1 } + +// Keys is how many rows the aggregator holds open. +func (a *Aggregator) Keys() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.core) + len(a.dims) + len(a.lateCore) + len(a.lateDims) +} + +// Count records n records of a property's day that were not counted, or with ReasonAccepted +// adds to the accepted ones. +func (a *Aggregator) Count(workspaceID, propertyID uuid.UUID, day time.Time, reason string, n int64) { + a.mu.Lock() + defer a.mu.Unlock() + a.init() + a.counts[countKey{workspaceID, propertyID, day, reason}] += n +} + +type resolved struct { + core CoreKey + dims [3]DimKey + n int +} + +// Add takes all of recs or, with ErrFull, none. Their values get their dictionary ids first, so a +// value past its cap is counted as (other) from the start. +func (a *Aggregator) Add(ctx context.Context, p Property, recs []Record) error { + if len(recs) == 0 { + return nil + } + now := a.Now() + var reqs []NameRequest + seen := map[NameRequest]int{} + ask := func(kind, value string, inferred bool) int { + if value == "" { + return -1 + } + k := NameRequest{Kind: kind, Value: value} + if i, ok := seen[k]; ok { + return i + } + seen[k] = len(reqs) + reqs = append(reqs, NameRequest{Kind: kind, Value: value, Inferred: inferred}) + return len(reqs) - 1 + } + type refs struct{ env, svc, route, ver, consumer, country, app int } + idx := make([]refs, len(recs)) + for i := range recs { + r := &recs[i] + idx[i] = refs{ask(KindEnvironment, r.Environment, false), ask(KindService, r.Service, false), ask(KindRoute, r.Route, r.Inferred), + ask(KindAPIVersion, r.APIVersion, false), ask(KindConsumer, r.Consumer, false), ask(KindCountry, r.Country, false), + ask(KindAppVersion, r.AppVersion, false)} + } + for _, kind := range []string{KindConsumer, KindCountry, KindAppVersion} { + ask(kind, OtherValue, false) + } + ids, err := a.names.resolve(ctx, a.Store, p, reqs, now) + if err != nil { + return err + } + id := func(i int) int32 { + if i < 0 { + return 0 + } + return ids[i] + } + other := map[int16]int32{} + for d, kind := range dimKinds { + other[d] = ids[seen[NameRequest{Kind: kind, Value: OtherValue}]] + } + keys := make([]resolved, len(recs)) + for i := range recs { + r, x := &recs[i], idx[i] + k := resolved{core: CoreKey{Environment: id(x.env), Service: id(x.svc), Route: id(x.route), APIVersion: id(x.ver), + Method: r.Method, Status: r.Status, Source: r.Source, Platform: r.Platform}} + for _, d := range []struct { + dim int16 + ref int + }{{DimConsumer, x.consumer}, {DimCountry, x.country}, {DimAppVersion, x.app}} { + if d.ref >= 0 { + k.dims[k.n] = DimKey{Dim: d.dim, Value: id(d.ref), Route: k.core.Route, Method: r.Method, StatusClass: StatusClass(r.Status)} + k.n++ + } + } + keys[i] = k + } + a.mu.Lock() + defer a.mu.Unlock() + a.init() + fresh := 0 + newCore, newDims := map[coreCellKey]bool{}, map[dimCellKey]bool{} + for i := range recs { + r := &recs[i] + minute := r.Time.UTC().Truncate(time.Minute) + ck := coreCellKey{p.WorkspaceID, p.ID, minute, keys[i].core} + if late := !minute.Add(time.Minute + LastFlush).After(now); late { + if a.lateCore[ck] == nil && !newCore[ck] { + newCore[ck], fresh = true, fresh+1 + } + } else if a.core[ck] == nil && !newCore[ck] { + newCore[ck], fresh = true, fresh+1 + } + hour := r.Hour.UTC() + for _, dk := range keys[i].dims[:keys[i].n] { + k := dimCellKey{p.WorkspaceID, p.ID, hour, dk} + if a.dims[k] == nil && a.lateDims[k] == nil && !newDims[k] { + newDims[k], fresh = true, fresh+1 + } + } + } + if a.MaxKeys > 0 && len(a.core)+len(a.dims)+len(a.lateCore)+len(a.lateDims)+fresh > a.MaxKeys { + return ErrFull + } + for i := range recs { + r := &recs[i] + minute := r.Time.UTC().Truncate(time.Minute) + hour, day := r.Hour.UTC(), r.Day + ck := coreCellKey{p.WorkspaceID, p.ID, minute, keys[i].core} + if !minute.Add(time.Minute + LastFlush).After(now) { + add(a.lateCore, ck, hour, day, time.Time{}, r) + } else { + add(a.core, ck, hour, day, minute.Add(time.Minute+LastFlush), r) + } + hk := hourKey{p.ID, hour} + late := !hour.Add(time.Hour + LastFlush).After(now) + for _, dk := range keys[i].dims[:keys[i].n] { + k := dimCellKey{p.WorkspaceID, p.ID, hour, dk} + if late { + add(a.lateDims, k, hour, day, time.Time{}, r) + continue + } + if a.dims[k] == nil { + if a.dimRows[hk] >= DimRowsPerHour { + k.key.Value = other[dk.Dim] + a.capped[hk] = true + } + if a.dims[k] == nil { + a.dimRows[hk]++ + } + } + add(a.dims, k, hour, day, hour.Add(time.Hour+LastFlush), r) + } + a.usage[p.WorkspaceID]++ + } + return nil +} + +func add[K comparable](m map[K]*cell, k K, hour, day, final time.Time, r *Record) { + c := m[k] + if c == nil { + c = &cell{hour: hour, day: day, agg: NewAgg(), final: final} + m[k] = c + } + c.agg.Add(r) + c.dirty = true +} + +// pending is what one flush took from memory, to put back when the write fails. +type pending struct { + w Write + core []coreCellKey + dims []dimCellKey + lateCore map[coreCellKey]*cell + lateDims map[dimCellKey]*cell + counts map[countKey]int64 + usage map[uuid.UUID]int64 + minuteDay []time.Time + hourMonth []time.Time +} + +func (a *Aggregator) take(now time.Time) *pending { + a.mu.Lock() + defer a.mu.Unlock() + a.init() + p := &pending{lateCore: a.lateCore, lateDims: a.lateDims, counts: a.counts, usage: a.usage} + a.lateCore, a.lateDims, a.counts, a.usage = map[coreCellKey]*cell{}, map[dimCellKey]*cell{}, map[countKey]int64{}, map[uuid.UUID]int64{} + hours := map[hourKey]*HourMark{} + mark := func(ws, prop uuid.UUID, hour, day time.Time) { + k := hourKey{prop, hour} + if hours[k] == nil { + hours[k] = &HourMark{WorkspaceID: ws, PropertyID: prop, Hour: hour, Day: day, Capped: a.capped[k]} + } + } + days, months := map[time.Time]bool{}, map[time.Time]bool{} + for k, c := range a.core { + if !c.dirty { + continue + } + c.dirty = false + p.core = append(p.core, k) + p.w.Minutes = append(p.w.Minutes, MinuteRow{WorkspaceID: k.ws, PropertyID: k.prop, Minute: k.minute, Hour: c.hour, Day: c.day, Writer: a.writer, Key: k.key, Agg: c.agg.Clone()}) + mark(k.ws, k.prop, c.hour, c.day) + days[utcDay(k.minute)] = true + } + for k, c := range a.dims { + if !c.dirty { + continue + } + c.dirty = false + if c.flushed && !now.Before(k.hour.Add(time.Hour+FoldAfter)) { + a.Log.Error("monitor: an hour's dimension rows were folded before this process wrote them a last time; its records since are lost", + slog.String("property_id", k.prop.String()), slog.Time("hour", k.hour)) + delete(a.dims, k) + continue + } + p.dims = append(p.dims, k) + p.w.Dims = append(p.w.Dims, DimRow{WorkspaceID: k.ws, PropertyID: k.prop, Hour: k.hour, Day: c.day, Writer: a.writer, Key: k.key, Agg: c.agg.Clone()}) + mark(k.ws, k.prop, c.hour, c.day) + months[monthOf(k.hour)] = true + } + if len(p.lateCore)+len(p.lateDims) > 0 { + late := newWriter() + for k, c := range p.lateCore { + p.w.Minutes = append(p.w.Minutes, MinuteRow{WorkspaceID: k.ws, PropertyID: k.prop, Minute: k.minute, Hour: c.hour, Day: c.day, Writer: late, Key: k.key, Agg: c.agg}) + mark(k.ws, k.prop, c.hour, c.day) + days[utcDay(k.minute)] = true + } + for k, c := range p.lateDims { + p.w.Dims = append(p.w.Dims, DimRow{WorkspaceID: k.ws, PropertyID: k.prop, Hour: k.hour, Day: c.day, Writer: late, Key: k.key, Agg: c.agg}) + mark(k.ws, k.prop, c.hour, c.day) + months[monthOf(k.hour)] = true + } + } + for _, h := range hours { + p.w.Hours = append(p.w.Hours, *h) + } + slices.SortFunc(p.w.Hours, func(x, y HourMark) int { return cmp.Or(x.PropertyID.Compare(y.PropertyID), x.Hour.Compare(y.Hour)) }) + for k, n := range p.counts { + p.w.Counts = append(p.w.Counts, Count{WorkspaceID: k.ws, PropertyID: k.prop, Day: k.day, Reason: k.reason, N: n}) + } + p.w.Usage = maps.Clone(p.usage) + p.minuteDay, p.hourMonth = slices.Collect(maps.Keys(days)), slices.Collect(maps.Keys(months)) + return p +} + +func (p *pending) empty() bool { + return len(p.w.Minutes) == 0 && len(p.w.Dims) == 0 && len(p.w.Counts) == 0 && len(p.w.Usage) == 0 +} + +func (a *Aggregator) putBack(p *pending) { + a.mu.Lock() + defer a.mu.Unlock() + for _, k := range p.core { + if c := a.core[k]; c != nil { + c.dirty = true + } + } + for _, k := range p.dims { + if c := a.dims[k]; c != nil { + c.dirty = true + } + } + for k, c := range p.lateCore { + if cur := a.lateCore[k]; cur != nil { + cur.agg.Merge(&c.agg) + } else { + a.lateCore[k] = c + } + } + for k, c := range p.lateDims { + if cur := a.lateDims[k]; cur != nil { + cur.agg.Merge(&c.agg) + } else { + a.lateDims[k] = c + } + } + for k, n := range p.counts { + a.counts[k] += n + } + for k, n := range p.usage { + a.usage[k] += n + } +} + +// forget drops the buckets that were written a last time and have not changed since. +func (a *Aggregator) forget(p *pending, now time.Time) { + a.mu.Lock() + defer a.mu.Unlock() + for _, k := range p.core { + if c := a.core[k]; c != nil { + c.flushed = true + } + } + for _, k := range p.dims { + if c := a.dims[k]; c != nil { + c.flushed = true + } + } + for k, c := range a.core { + if !c.dirty && !now.Before(c.final) { + delete(a.core, k) + } + } + for k, c := range a.dims { + if !c.dirty && !now.Before(c.final) { + delete(a.dims, k) + } + } + for k := range a.dimRows { + if !now.Before(k.hour.Add(time.Hour + LastFlush)) { + delete(a.dimRows, k) + delete(a.capped, k) + } + } +} + +// Prepare makes the minute partitions of yesterday to two days ahead (UTC) and the hour partitions +// of this month and the next exist. +func (a *Aggregator) Prepare(ctx context.Context) error { + today := utcDay(a.Now()) + if err := a.ensure(ctx, []time.Time{today.AddDate(0, 0, -1), today, today.AddDate(0, 0, 1), today.AddDate(0, 0, 2)}, nil); err != nil { + return err + } + m := monthOf(today) + return a.ensure(ctx, nil, []time.Time{m.AddDate(0, -1, 0), m, m.AddDate(0, 1, 0)}) +} + +func (a *Aggregator) ensure(ctx context.Context, days, months []time.Time) error { + a.partMu.Lock() + defer a.partMu.Unlock() + if a.minuteDay == nil { + a.minuteDay, a.hourMonth = map[time.Time]bool{}, map[time.Time]bool{} + } + for _, d := range days { + if !a.minuteDay[d] { + if err := a.Store.EnsureMinutePartitions(ctx, d, d); err != nil { + return err + } + a.minuteDay[d] = true + } + } + for _, m := range months { + if !a.hourMonth[m] { + if err := a.Store.EnsureHourPartitions(ctx, m, m); err != nil { + return err + } + a.hourMonth[m] = true + } + } + return nil +} + +// Flush writes every bucket that changed since the last flush; on failure everything stays for the +// next one. +func (a *Aggregator) Flush(ctx context.Context) error { + now := a.Now() + p := a.take(now) + if p.empty() { + a.forget(p, now) + return nil + } + err := a.ensure(ctx, p.minuteDay, p.hourMonth) + if err == nil { + err = a.Store.Write(ctx, p.w, now) + } + if err != nil { + a.putBack(p) + return err + } + a.forget(p, now) + return nil +} + +// Run flushes every ten seconds until ctx ends, then writes everything left for up to ten seconds. +func (a *Aggregator) Run(ctx context.Context) { + tick := time.NewTicker(FlushInterval) + defer tick.Stop() + backoff := time.Duration(0) + var retryAt time.Time + for { + select { + case <-ctx.Done(): + a.drain() + return + case <-tick.C: + } + if time.Now().Before(retryAt) { + continue + } + if err := a.Flush(ctx); err != nil { + backoff = min(max(backoff*2, FlushInterval), maxBackoff) + retryAt = time.Now().Add(backoff) + a.Log.ErrorContext(ctx, "monitor flush", slog.Any("error", err), slog.Duration("retry_in", backoff)) + continue + } + backoff = 0 + } +} + +func (a *Aggregator) drain() { + ctx, cancel := context.WithTimeout(context.Background(), drainTimeout) + defer cancel() + for ctx.Err() == nil { + err := a.Flush(ctx) + if err == nil { + return + } + a.Log.Error("monitor drain", slog.Any("error", err), slog.Int("rows_left", a.Keys())) + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } +} + +// Forget drops what the aggregator knows of a property's names, after it was deleted or changed. +func (a *Aggregator) Forget(propertyID uuid.UUID) { a.names.forget(propertyID) } + +type nameKey struct { + kind, value string +} + +type propNames struct { + ids map[nameKey]int32 + full map[string]bool + loaded time.Time +} + +// nameCache keeps a property's dictionary ids for ten minutes. A kind whose cap is reached maps +// every value it does not know to (other) without asking again. +type nameCache struct { + mu sync.Mutex + props map[uuid.UUID]*propNames + size int +} + +func (c *nameCache) forget(propertyID uuid.UUID) { + c.mu.Lock() + defer c.mu.Unlock() + if pn := c.props[propertyID]; pn != nil { + c.size -= len(pn.ids) + delete(c.props, propertyID) + } +} + +func (c *nameCache) resolve(ctx context.Context, st Store, p Property, reqs []NameRequest, now time.Time) ([]int32, error) { + out := make([]int32, len(reqs)) + var miss []int + c.mu.Lock() + if c.props == nil || c.size > maxNames { + c.props, c.size = map[uuid.UUID]*propNames{}, 0 + } + pn := c.props[p.ID] + if pn == nil || now.Sub(pn.loaded) > namesTTL || now.Before(pn.loaded) { + if pn != nil { + c.size -= len(pn.ids) + } + pn = &propNames{ids: map[nameKey]int32{}, full: map[string]bool{}, loaded: now} + c.props[p.ID] = pn + } + for i, r := range reqs { + if id, ok := pn.ids[nameKey{r.Kind, r.Value}]; ok { + out[i] = id + continue + } + if other, ok := pn.ids[nameKey{r.Kind, OtherValue}]; ok && pn.full[r.Kind] { + out[i] = other + continue + } + miss = append(miss, i) + } + c.mu.Unlock() + if len(miss) == 0 { + return out, nil + } + ask := make([]NameRequest, 0, len(miss)+1) + kinds := map[string]bool{} + for _, i := range miss { + ask = append(ask, reqs[i]) + kinds[reqs[i].Kind] = true + } + others := map[string]int{} + for _, kind := range slices.Sorted(maps.Keys(kinds)) { + if _, capped := Caps[kind]; capped { + others[kind] = len(ask) + ask = append(ask, NameRequest{Kind: kind, Value: OtherValue}) + } + } + ids, err := st.Names(ctx, p.WorkspaceID, p.ID, ask, now) + if err != nil { + return nil, err + } + c.mu.Lock() + defer c.mu.Unlock() + for kind, at := range others { + if _, ok := pn.ids[nameKey{kind, OtherValue}]; !ok { + c.size++ + } + pn.ids[nameKey{kind, OtherValue}] = ids[at] + } + for j, i := range miss { + r := reqs[i] + out[i] = ids[j] + if at, ok := others[r.Kind]; ok && ids[j] == ids[at] && r.Value != OtherValue { + pn.full[r.Kind] = true + continue + } + if _, ok := pn.ids[nameKey{r.Kind, r.Value}]; !ok { + c.size++ + } + pn.ids[nameKey{r.Kind, r.Value}] = ids[j] + } + return out, nil +} + +func utcDay(t time.Time) time.Time { + y, m, d := t.UTC().Date() + return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) +} + +func monthOf(t time.Time) time.Time { + y, m, _ := t.UTC().Date() + return time.Date(y, m, 1, 0, 0, 0, 0, time.UTC) +} diff --git a/api/internal/monitor/fold.go b/api/internal/monitor/fold.go new file mode 100644 index 00000000..5d8d92d1 --- /dev/null +++ b/api/internal/monitor/fold.go @@ -0,0 +1,340 @@ +package monitor + +import ( + "cmp" + "context" + "fmt" + "slices" + "time" + "uuid" + + "github.com/jackc/pgx/v5" +) + +// aggScan reads the aggColumns of a row. +type aggScan struct { + agg Agg + durations, sizes []byte +} + +func (a *aggScan) dest() []any { + g := &a.agg + return []any{&g.Requests, &g.Errors, &g.ClientErrors, &g.Sampled, &g.DurationSum, &g.DurationMin, &g.DurationMax, &a.durations, + &g.RequestBytes, &g.ResponseBytes, &a.sizes} +} + +func (a *aggScan) decode() (Agg, error) { + var err error + if a.agg.Durations, err = DecodeSketch(a.durations); err != nil { + return Agg{}, err + } + if a.agg.Sizes, err = DecodeSketch(a.sizes); err != nil { + return Agg{}, err + } + return a.agg, nil +} + +func aggValues(g *Agg) []any { + return []any{g.Requests, g.Errors, g.ClientErrors, g.Sampled, g.DurationSum, g.DurationMin, g.DurationMax, g.Durations.Encode(), + g.RequestBytes, g.ResponseBytes, g.Sizes.Encode()} +} + +var aggCopyColumns = []string{"requests", "errors", "client_errors", "sampled", "duration_sum", "duration_min", "duration_max", "durations", + "request_bytes", "response_bytes", "response_sizes"} + +const coreColumns = "environment, service, route, method, status, api_version, source, platform" + +var coreCopyColumns = []string{"environment", "service", "route", "method", "status", "api_version", "source", "platform"} + +func coreDest(k *CoreKey) []any { + return []any{&k.Environment, &k.Service, &k.Route, &k.Method, &k.Status, &k.APIVersion, &k.Source, &k.Platform} +} + +func coreValues(k CoreKey) []any { + return []any{k.Environment, k.Service, k.Route, k.Method, k.Status, k.APIVersion, int16(k.Source), int16(k.Platform)} +} + +const dimColumns = "dim, value, route, method, status_class" + +var dimCopyColumns = []string{"dim", "value", "route", "method", "status_class"} + +func dimDest(k *DimKey) []any { return []any{&k.Dim, &k.Value, &k.Route, &k.Method, &k.StatusClass} } + +func dimValues(k DimKey) []any { return []any{k.Dim, k.Value, k.Route, k.Method, k.StatusClass} } + +func collectCore(rows pgx.Rows) (map[CoreKey]*Agg, error) { + defer rows.Close() + out := map[CoreKey]*Agg{} + for rows.Next() { + var k CoreKey + var a aggScan + if err := rows.Scan(append(coreDest(&k), a.dest()...)...); err != nil { + return nil, err + } + g, err := a.decode() + if err != nil { + return nil, err + } + if cur := out[k]; cur != nil { + cur.Merge(&g) + } else { + out[k] = &g + } + } + return out, rows.Err() +} + +func collectDims(rows pgx.Rows) (map[DimKey]*Agg, error) { + defer rows.Close() + out := map[DimKey]*Agg{} + for rows.Next() { + var k DimKey + var a aggScan + if err := rows.Scan(append(dimDest(&k), a.dest()...)...); err != nil { + return nil, err + } + g, err := a.decode() + if err != nil { + return nil, err + } + if cur := out[k]; cur != nil { + cur.Merge(&g) + } else { + out[k] = &g + } + } + return out, rows.Err() +} + +func sortedCore(m map[CoreKey]*Agg) []CoreKey { + keys := make([]CoreKey, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.SortFunc(keys, compareCore) + return keys +} + +func sortedDims(m map[DimKey]*Agg) []DimKey { + keys := make([]DimKey, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.SortFunc(keys, compareDim) + return keys +} + +// capDims keeps an hour's top dimension rows by requests and adds the rest up under each +// dimension's (other); it says whether it had to. +func capDims(ctx context.Context, tx pgx.Tx, ws, prop uuid.UUID, rows map[DimKey]*Agg, now time.Time) (bool, error) { + if len(rows) <= DimRowsPerHour { + return false, nil + } + keys := sortedDims(rows) + slices.SortStableFunc(keys, func(a, b DimKey) int { return cmp.Compare(rows[b].Requests, rows[a].Requests) }) + other := map[int16]int32{} + for d, kind := range dimKinds { + ids, err := names(ctx, tx, ws, prop, kind, []string{OtherValue}, nil, now) + if err != nil { + return false, err + } + other[d] = ids[OtherValue] + } + for _, k := range keys[DimRowsPerHour:] { + g := rows[k] + delete(rows, k) + o := k + o.Value = other[k.Dim] + if cur := rows[o]; cur != nil { + cur.Merge(g) + } else { + rows[o] = g + } + } + return true, nil +} + +func bumpVersion(ctx context.Context, tx pgx.Tx, ws, prop uuid.UUID) error { + _, err := tx.Exec(ctx, `UPDATE analytics_properties SET rollup_version = rollup_version + 1 WHERE workspace_id = $1 AND id = $2`, ws, prop) + return err +} + +// FoldHour folds a closed local hour: its minute rows into one hourly row per key, and its +// dimension rows of every writer into one. The hour's row in monitor_rollup_hours is locked first, +// so a writer that adds rows meanwhile marks the hour dirty again rather than being lost. +func (s *PGStore) FoldHour(ctx context.Context, ws, prop uuid.UUID, hour time.Time, now time.Time) error { + if err := s.EnsureHourPartitions(ctx, hour, hour); err != nil { + return err + } + return pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error { + var capped bool + if err := tx.QueryRow(ctx, `SELECT capped FROM monitor_rollup_hours WHERE workspace_id = $1 AND property_id = $2 AND hour = $3 FOR UPDATE`, + ws, prop, hour).Scan(&capped); err != nil { + return err + } + rows, err := tx.Query(ctx, `SELECT `+coreColumns+`, `+aggColumns+` FROM monitor_minute + WHERE workspace_id = $1 AND property_id = $2 AND minute >= $3 AND minute < $4`, ws, prop, hour, hour.Add(time.Hour)) + if err != nil { + return err + } + core, err := collectCore(rows) + if err != nil { + return fmt.Errorf("fold hour: %w", err) + } + if _, err := tx.Exec(ctx, `DELETE FROM monitor_hour WHERE workspace_id = $1 AND property_id = $2 AND hour = $3`, ws, prop, hour); err != nil { + return err + } + copyRows := make([][]any, 0, len(core)) + for _, k := range sortedCore(core) { + copyRows = append(copyRows, append(append([]any{ws, prop, hour}, coreValues(k)...), aggValues(core[k])...)) + } + if _, err := tx.CopyFrom(ctx, pgx.Identifier{HourTable}, append(append([]string{"workspace_id", "property_id", "hour"}, coreCopyColumns...), aggCopyColumns...), + pgx.CopyFromRows(copyRows)); err != nil { + return fmt.Errorf("fold hour: %w", err) + } + rows, err = tx.Query(ctx, `DELETE FROM monitor_dim_hour WHERE workspace_id = $1 AND property_id = $2 AND hour = $3 + RETURNING `+dimColumns+`, `+aggColumns, ws, prop, hour) + if err != nil { + return err + } + dims, err := collectDims(rows) + if err != nil { + return fmt.Errorf("fold hour: %w", err) + } + over, err := capDims(ctx, tx, ws, prop, dims, now) + if err != nil { + return err + } + copyRows = make([][]any, 0, len(dims)) + for _, k := range sortedDims(dims) { + copyRows = append(copyRows, append(append([]any{ws, prop, hour, int64(0)}, dimValues(k)...), aggValues(dims[k])...)) + } + if _, err := tx.CopyFrom(ctx, pgx.Identifier{DimHourTable}, append(append([]string{"workspace_id", "property_id", "hour", "writer"}, dimCopyColumns...), aggCopyColumns...), + pgx.CopyFromRows(copyRows)); err != nil { + return fmt.Errorf("fold hour: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE monitor_rollup_hours SET built_at = $4, dirty = false, day_built_at = NULL, capped = capped OR $5 + WHERE workspace_id = $1 AND property_id = $2 AND hour = $3`, ws, prop, hour, now, over); err != nil { + return err + } + return bumpVersion(ctx, tx, ws, prop) + }) +} + +// FoldDay folds a finished local day from its hours, once each of them is folded; it reports +// whether it did. +func (s *PGStore) FoldDay(ctx context.Context, ws, prop uuid.UUID, day time.Time, now time.Time) (bool, error) { + done := false + err := pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error { + rows, err := tx.Query(ctx, `SELECT hour, built_at IS NOT NULL AND NOT dirty FROM monitor_rollup_hours + WHERE workspace_id = $1 AND property_id = $2 AND day = $3 ORDER BY hour FOR UPDATE`, ws, prop, day) + if err != nil { + return err + } + var hours []time.Time + ready := true + for rows.Next() { + var h time.Time + var built bool + if err := rows.Scan(&h, &built); err != nil { + rows.Close() + return err + } + hours, ready = append(hours, h), ready && built + } + rows.Close() + if err := rows.Err(); err != nil || !ready || len(hours) == 0 { + return err + } + rows, err = tx.Query(ctx, `SELECT `+coreColumns+`, `+aggColumns+` FROM monitor_hour + WHERE workspace_id = $1 AND property_id = $2 AND hour = ANY($3)`, ws, prop, hours) + if err != nil { + return err + } + core, err := collectCore(rows) + if err != nil { + return fmt.Errorf("fold day: %w", err) + } + rows, err = tx.Query(ctx, `SELECT `+dimColumns+`, `+aggColumns+` FROM monitor_dim_hour + WHERE workspace_id = $1 AND property_id = $2 AND hour = ANY($3)`, ws, prop, hours) + if err != nil { + return err + } + dims, err := collectDims(rows) + if err != nil { + return fmt.Errorf("fold day: %w", err) + } + for _, t := range []string{"monitor_day", "monitor_dim_day"} { + if _, err := tx.Exec(ctx, "DELETE FROM "+t+" WHERE workspace_id = $1 AND property_id = $2 AND day = $3", ws, prop, day); err != nil { + return err + } + } + copyRows := make([][]any, 0, len(core)) + for _, k := range sortedCore(core) { + copyRows = append(copyRows, append(append([]any{ws, prop, day}, coreValues(k)...), aggValues(core[k])...)) + } + if _, err := tx.CopyFrom(ctx, pgx.Identifier{"monitor_day"}, append(append([]string{"workspace_id", "property_id", "day"}, coreCopyColumns...), aggCopyColumns...), + pgx.CopyFromRows(copyRows)); err != nil { + return fmt.Errorf("fold day: %w", err) + } + copyRows = make([][]any, 0, len(dims)) + for _, k := range sortedDims(dims) { + copyRows = append(copyRows, append(append([]any{ws, prop, day}, dimValues(k)...), aggValues(dims[k])...)) + } + if _, err := tx.CopyFrom(ctx, pgx.Identifier{"monitor_dim_day"}, append(append([]string{"workspace_id", "property_id", "day"}, dimCopyColumns...), aggCopyColumns...), + pgx.CopyFromRows(copyRows)); err != nil { + return fmt.Errorf("fold day: %w", err) + } + if _, err := tx.Exec(ctx, `UPDATE monitor_rollup_hours SET day_built_at = $4 WHERE workspace_id = $1 AND property_id = $2 AND day = $3`, + ws, prop, day, now); err != nil { + return err + } + done = true + return bumpVersion(ctx, tx, ws, prop) + }) + return done, err +} + +// Rollup folds a property's hours that closed FoldAfter ago and are not folded or changed since, +// then its local days that ended FoldAfter ago with every hour folded; it says how many of each. +func (s *PGStore) Rollup(ctx context.Context, ws, prop uuid.UUID, loc *time.Location, now time.Time) (int, int, error) { + rows, err := s.Pool.Query(ctx, `SELECT hour FROM monitor_rollup_hours + WHERE workspace_id = $1 AND property_id = $2 AND (built_at IS NULL OR dirty) AND hour <= $3 ORDER BY hour`, + ws, prop, now.Add(-time.Hour-FoldAfter)) + if err != nil { + return 0, 0, err + } + hours, err := pgx.CollectRows(rows, pgx.RowTo[time.Time]) + if err != nil { + return 0, 0, err + } + for _, h := range hours { + if err := s.FoldHour(ctx, ws, prop, h, now); err != nil { + return 0, 0, err + } + } + rows, err = s.Pool.Query(ctx, `SELECT day FROM monitor_rollup_hours WHERE workspace_id = $1 AND property_id = $2 + GROUP BY day HAVING bool_and(built_at IS NOT NULL AND NOT dirty) AND bool_or(day_built_at IS NULL) ORDER BY day`, ws, prop) + if err != nil { + return len(hours), 0, err + } + days, err := pgx.CollectRows(rows, pgx.RowTo[time.Time]) + if err != nil { + return len(hours), 0, err + } + folded := 0 + for _, d := range days { + d = d.UTC() + if now.Before(DayStart(d.AddDate(0, 0, 1), loc).Add(FoldAfter)) { + continue + } + ok, err := s.FoldDay(ctx, ws, prop, d, now) + if err != nil { + return len(hours), folded, err + } + if ok { + folded++ + } + } + return len(hours), folded, nil +} diff --git a/api/internal/monitor/maintain.go b/api/internal/monitor/maintain.go new file mode 100644 index 00000000..8379f7e4 --- /dev/null +++ b/api/internal/monitor/maintain.go @@ -0,0 +1,76 @@ +package monitor + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/productdevbook/yuva/api/internal/analytics" +) + +var ( + minutePartition = regexp.MustCompile(`^monitor_minute_(\d{8})$`) + hourPartition = regexp.MustCompile(`^monitor_(?:dim_)?hour_(\d{6})$`) +) + +// DropPartitionsBefore drops the minute partitions of UTC days before day and the hour partitions +// of months before month. It is server maintenance over every workspace's rows of those days (the +// exception to the workspace rule of analytics_partitions, see architecture.md). +func (s *PGStore) DropPartitionsBefore(ctx context.Context, day, month time.Time) ([]string, error) { + conn, err := s.Pool.Acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", int64(analytics.PartitionLock)); err != nil { + return nil, err + } + defer conn.Exec(context.WithoutCancel(ctx), "SELECT pg_advisory_unlock($1)", int64(analytics.PartitionLock)) + rows, err := conn.Query(ctx, `SELECT c.relname, p.relname, i.inhdetachpending FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_class p ON p.oid = i.inhparent + WHERE p.relname IN ('monitor_minute', 'monitor_hour', 'monitor_dim_hour') ORDER BY c.relname`) + if err != nil { + return nil, err + } + type part struct { + name, parent string + pending bool + } + parts, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (part, error) { + var p part + err := r.Scan(&p.name, &p.parent, &p.pending) + return p, err + }) + if err != nil { + return nil, err + } + var dropped []string + for _, p := range parts { + var old bool + if m := minutePartition.FindStringSubmatch(p.name); m != nil { + d, err := time.Parse("20060102", m[1]) + old = err == nil && d.Before(day) + } else if m := hourPartition.FindStringSubmatch(p.name); m != nil { + d, err := time.Parse("200601", m[1]) + old = err == nil && d.Before(month) + } + if !old { + continue + } + detach := "CONCURRENTLY" + if p.pending { + detach = "FINALIZE" + } + if _, err := conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s DETACH PARTITION %s %s", pgx.Identifier{p.parent}.Sanitize(), pgx.Identifier{p.name}.Sanitize(), detach)); err != nil { + return dropped, fmt.Errorf("detach %s: %w", p.name, err) + } + if _, err := conn.Exec(ctx, "DROP TABLE "+pgx.Identifier{p.name}.Sanitize()); err != nil { + return dropped, fmt.Errorf("drop %s: %w", p.name, err) + } + dropped = append(dropped, p.name) + } + return dropped, nil +} diff --git a/api/internal/monitor/monitor.go b/api/internal/monitor/monitor.go new file mode 100644 index 00000000..815a7fde --- /dev/null +++ b/api/internal/monitor/monitor.go @@ -0,0 +1,274 @@ +// Package monitor counts an API's requests: records are added up in memory per minute and hour and +// written as rows of counters and duration sketches, never a row per request. Handlers depend on +// the Store interface, never on its tables. +package monitor + +import ( + "context" + "time" + "uuid" +) + +type Source int16 + +const ( + SourceServer Source = 1 + SourceOTLP Source = 2 + SourceApp Source = 3 +) + +type Platform int16 + +const ( + PlatformNone Platform = 0 + PlatformIOS Platform = 1 + PlatformAndroid Platform = 2 +) + +// Extra dimensions, each kept by local hour with route, method and status class. +const ( + DimConsumer int16 = 1 + DimCountry int16 = 2 + DimAppVersion int16 = 3 +) + +// Name kinds of the dictionaries and the caps on them. +const ( + KindRoute = "route" + KindEnvironment = "environment" + KindService = "service" + KindAPIVersion = "api_version" + KindConsumer = "consumer" + KindAppVersion = "app_version" + KindCountry = "country" +) + +var Caps = map[string]int{ + KindRoute: 1000, KindConsumer: 10_000, KindAPIVersion: 50, KindAppVersion: 200, KindEnvironment: 20, KindService: 100, +} + +// DimRowsPerHour is how many extra-dimension rows a property's hour holds; later new combinations +// count as (other). +const DimRowsPerHour = 50_000 + +var dimKinds = map[int16]string{DimConsumer: KindConsumer, DimCountry: KindCountry, DimAppVersion: KindAppVersion} + +// DefaultEnvironment is a record's environment when it names none. +const DefaultEnvironment = "production" + +// Drop reasons, as counted per property and day. +const ( + ReasonAccepted = "accepted" + ReasonInvalid = "invalid" + ReasonLate = "late" + ReasonRate = "rate" + ReasonOverLimit = "over_limit" + ReasonBuffer = "buffer" +) + +// Record is one request once checked. Hour is the local hour the request started in and Day that +// hour's local day; Weight is how many requests it stands for. +type Record struct { + Time time.Time + Hour time.Time + Day time.Time + Environment string + Service string + Route string + Inferred bool + Method int16 + Status int16 + APIVersion string + Source Source + Platform Platform + DurationUS int64 + // RequestBytes and ResponseBytes are -1 when unknown. + RequestBytes int64 + ResponseBytes int64 + Consumer string + Country string + AppVersion string + Weight int64 +} + +// LocalHour is the first instant of the local hour t is in: the hour starts when loc's clock last +// showed a full hour, so both of the hours a clock repeats in autumn are hours of their own. +func LocalHour(t time.Time, loc *time.Location) time.Time { + lt := t.In(loc) + return t.Add(-time.Duration(lt.Minute())*time.Minute - time.Duration(lt.Second())*time.Second - time.Duration(lt.Nanosecond())).UTC() +} + +// LocalDay is the date of t in loc, as midnight UTC. +func LocalDay(t time.Time, loc *time.Location) time.Time { + y, m, d := t.In(loc).Date() + return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) +} + +// DayStart is when a local day begins. +func DayStart(day time.Time, loc *time.Location) time.Time { + return time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, loc).UTC() +} + +// Property is what aggregation needs of an API property. +type Property struct { + WorkspaceID uuid.UUID + ID uuid.UUID + Location *time.Location +} + +// Agg is what rows add up: exact counters and the duration and response size sketches. +type Agg struct { + Requests int64 + Errors int64 + ClientErrors int64 + Sampled int64 + DurationSum int64 + DurationMin int64 + DurationMax int64 + Durations Sketch + RequestBytes int64 + ResponseBytes int64 + Sizes Sketch +} + +func NewAgg() Agg { return Agg{Durations: NewSketch(DurationScale), Sizes: NewSketch(SizeScale)} } + +func (a *Agg) Add(r *Record) { + w := max(r.Weight, 1) + if a.Requests == 0 || r.DurationUS < a.DurationMin { + a.DurationMin = r.DurationUS + } + a.DurationMax = max(a.DurationMax, r.DurationUS) + a.Requests += w + if w > 1 { + a.Sampled += w + } + if IsError(r.Status) { + a.Errors += w + } else if IsClientError(r.Status) { + a.ClientErrors += w + } + a.DurationSum += r.DurationUS * w + a.Durations.Add(r.DurationUS, uint64(w)) + if r.RequestBytes > 0 { + a.RequestBytes += r.RequestBytes * w + } + if r.ResponseBytes >= 0 { + a.ResponseBytes += r.ResponseBytes * w + a.Sizes.Add(r.ResponseBytes, uint64(w)) + } +} + +func (a *Agg) Merge(b *Agg) { + if b.Requests == 0 { + return + } + if a.Requests == 0 || b.DurationMin < a.DurationMin { + a.DurationMin = b.DurationMin + } + a.DurationMax = max(a.DurationMax, b.DurationMax) + a.Requests += b.Requests + a.Errors += b.Errors + a.ClientErrors += b.ClientErrors + a.Sampled += b.Sampled + a.DurationSum += b.DurationSum + a.RequestBytes += b.RequestBytes + a.ResponseBytes += b.ResponseBytes + a.Durations.Merge(b.Durations) + a.Sizes.Merge(b.Sizes) +} + +func (a *Agg) Clone() Agg { + out := *a + out.Durations, out.Sizes = a.Durations.Clone(), a.Sizes.Clone() + return out +} + +// DurationQuantile is a duration percentile in microseconds, kept within the exact minimum and +// maximum. +func (a *Agg) DurationQuantile(q float64) float64 { + if a.Requests == 0 { + return 0 + } + return min(max(a.Durations.Quantile(q), float64(a.DurationMin)), float64(a.DurationMax)) +} + +// CoreKey is a minute row's key, with the dictionaries' ids. +type CoreKey struct { + Environment, Service, Route, APIVersion int32 + Method, Status int16 + Source Source + Platform Platform +} + +// DimKey is an extra dimension row's key. +type DimKey struct { + Dim int16 + Value int32 + Route int32 + Method int16 + StatusClass int16 +} + +// MinuteRow is a writer's cumulative totals of a key in a UTC minute; Hour and Day place the minute +// in the property's local hour and day. +type MinuteRow struct { + WorkspaceID, PropertyID uuid.UUID + Minute, Hour, Day time.Time + Writer int64 + Key CoreKey + Agg Agg +} + +type DimRow struct { + WorkspaceID, PropertyID uuid.UUID + Hour, Day time.Time + Writer int64 + Key DimKey + Agg Agg +} + +// Count is a number of records of a property's day for one reason. +type Count struct { + WorkspaceID, PropertyID uuid.UUID + Day time.Time + Reason string + N int64 +} + +// HourMark is a local hour a write touched; Capped says it reached its dimension rows. +type HourMark struct { + WorkspaceID, PropertyID uuid.UUID + Hour, Day time.Time + Capped bool +} + +// Write is one flush: rows, the hours they touched, ingest counts and each workspace's new records. +type Write struct { + Minutes []MinuteRow + Dims []DimRow + Hours []HourMark + Counts []Count + Usage map[uuid.UUID]int64 +} + +// NameRequest asks for the id of a value of a kind; Inferred applies to routes. +type NameRequest struct { + Kind string + Value string + Inferred bool +} + +type Store interface { + // EnsureMinutePartitions makes the minute partitions of the UTC days from to to exist; + // EnsureHourPartitions the hour partitions of the months of from to to. + EnsureMinutePartitions(ctx context.Context, from, to time.Time) error + EnsureHourPartitions(ctx context.Context, from, to time.Time) error + // Names returns the ids of values of a property, adding the new ones as far as their caps + // allow and giving the others (other)'s id. + Names(ctx context.Context, workspaceID, propertyID uuid.UUID, reqs []NameRequest, now time.Time) ([]int32, error) + // Write upserts a flush in one transaction. + Write(ctx context.Context, w Write, now time.Time) error + // Query answers one query over a property's rows. + Query(ctx context.Context, q Query) (Result, error) +} diff --git a/api/internal/monitor/pgstore.go b/api/internal/monitor/pgstore.go new file mode 100644 index 00000000..196f34ef --- /dev/null +++ b/api/internal/monitor/pgstore.go @@ -0,0 +1,475 @@ +package monitor + +import ( + "cmp" + "context" + "fmt" + "maps" + "slices" + "strings" + "time" + "uuid" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/productdevbook/yuva/api/internal/analytics" +) + +const ( + MinuteTable = "monitor_minute" + HourTable = "monitor_hour" + DimHourTable = "monitor_dim_hour" + namesLock = 7_310_016 +) + +// PGStore keeps an API's rows in Postgres. +type PGStore struct { + Pool *pgxpool.Pool +} + +var _ Store = (*PGStore)(nil) + +// MinutePartition is the partition of monitor_minute that holds one UTC day; HourPartition the +// partition of an hourly table that holds one month. +func MinutePartition(day time.Time) string { return MinuteTable + "_" + day.Format("20060102") } + +func HourPartition(table string, month time.Time) string { return table + "_" + month.Format("200601") } + +func (s *PGStore) EnsureMinutePartitions(ctx context.Context, from, to time.Time) error { + var parts []partition + for d := utcDay(from); !d.After(utcDay(to)); d = d.AddDate(0, 0, 1) { + parts = append(parts, partition{MinuteTable, MinutePartition(d), d, d.AddDate(0, 0, 1)}) + } + return s.ensure(ctx, parts) +} + +func (s *PGStore) EnsureHourPartitions(ctx context.Context, from, to time.Time) error { + var parts []partition + for m := monthOf(from); !m.After(monthOf(to)); m = m.AddDate(0, 1, 0) { + for _, t := range []string{HourTable, DimHourTable} { + parts = append(parts, partition{t, HourPartition(t, m), m, m.AddDate(0, 1, 0)}) + } + } + return s.ensure(ctx, parts) +} + +type partition struct { + parent, name string + from, to time.Time +} + +func (s *PGStore) ensure(ctx context.Context, parts []partition) error { + return pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock($1)", int64(analytics.PartitionLock)); err != nil { + return err + } + for _, p := range parts { + var exists bool + if err := tx.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", p.name).Scan(&exists); err != nil { + return err + } + if exists { + continue + } + if _, err := tx.Exec(ctx, fmt.Sprintf("CREATE TABLE %s PARTITION OF %s FOR VALUES FROM ('%s') TO ('%s') WITH (fillfactor = 70)", + p.name, p.parent, p.from.Format(time.RFC3339), p.to.Format(time.RFC3339))); err != nil { + return fmt.Errorf("create partition %s: %w", p.name, err) + } + } + return nil + }) +} + +// Names finds or adds each value's id under a lock per property and kind, so two processes never +// pass a cap together. (other) and (unmatched) never count towards a cap; services are capped per +// workspace. +func (s *PGStore) Names(ctx context.Context, ws, prop uuid.UUID, reqs []NameRequest, now time.Time) ([]int32, error) { + out := make([]int32, len(reqs)) + byKind := map[string][]int{} + for i, r := range reqs { + byKind[r.Kind] = append(byKind[r.Kind], i) + } + err := pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error { + for _, kind := range slices.Sorted(maps.Keys(byKind)) { + idx := byKind[kind] + values := make([]string, 0, len(idx)) + for _, i := range idx { + values = append(values, reqs[i].Value) + } + ids, err := names(ctx, tx, ws, prop, kind, values, inferredOf(reqs, idx), now) + if err != nil { + return err + } + for _, i := range idx { + out[i] = ids[reqs[i].Value] + } + } + return nil + }) + return out, err +} + +func inferredOf(reqs []NameRequest, idx []int) map[string]bool { + out := map[string]bool{} + for _, i := range idx { + out[reqs[i].Value] = reqs[i].Inferred + } + return out +} + +func special(v string) bool { return v == OtherValue || v == Unmatched } + +func names(ctx context.Context, tx pgx.Tx, ws, prop uuid.UUID, kind string, values []string, inferred map[string]bool, now time.Time) (map[string]int32, error) { + table, col, where := "monitor_names", "value", " AND kind = $3" + args := []any{ws, prop, kind} + if kind == KindRoute { + table, col, where = "monitor_routes", "pattern", "" + args = args[:2] + } + find := func(vs []string) (map[string]int32, error) { + rows, err := tx.Query(ctx, fmt.Sprintf("SELECT %s, id FROM %s WHERE workspace_id = $1 AND property_id = $2%s AND %s = ANY($%d)", + col, table, where, col, len(args)+1), append(slices.Clone(args), vs)...) + if err != nil { + return nil, err + } + out := map[string]int32{} + for rows.Next() { + var v string + var id int32 + if err := rows.Scan(&v, &id); err != nil { + rows.Close() + return nil, err + } + out[v] = id + } + rows.Close() + return out, rows.Err() + } + got, err := find(values) + if err != nil { + return nil, err + } + var missing []string + for _, v := range values { + if _, ok := got[v]; !ok && !slices.Contains(missing, v) { + missing = append(missing, v) + } + } + if len(missing) == 0 { + return got, nil + } + if _, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock($1, hashtext($2))", int32(namesLock), prop.String()+"/"+kind); err != nil { + return nil, err + } + if got, err = find(values); err != nil { + return nil, err + } + missing = slices.DeleteFunc(missing, func(v string) bool { _, ok := got[v]; return ok }) + if len(missing) == 0 { + return got, nil + } + var used int + switch kind { + case KindRoute: + err = tx.QueryRow(ctx, `SELECT count(*) FROM monitor_routes WHERE workspace_id = $1 AND property_id = $2 AND pattern NOT IN ('(other)', '(unmatched)')`, + ws, prop).Scan(&used) + case KindService: + err = tx.QueryRow(ctx, `SELECT count(DISTINCT value) FROM monitor_names WHERE workspace_id = $1 AND kind = 'service' AND value <> '(other)'`, ws).Scan(&used) + default: + err = tx.QueryRow(ctx, `SELECT count(*) FROM monitor_names WHERE workspace_id = $1 AND property_id = $2 AND kind = $3 AND value <> '(other)'`, + ws, prop, kind).Scan(&used) + } + if err != nil { + return nil, err + } + var elsewhere map[string]bool + if kind == KindService { + rows, err := tx.Query(ctx, `SELECT DISTINCT value FROM monitor_names WHERE workspace_id = $1 AND kind = 'service' AND value = ANY($2)`, ws, missing) + if err != nil { + return nil, err + } + list, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, err + } + elsewhere = map[string]bool{} + for _, v := range list { + elsewhere[v] = true + } + } + var next int32 + if err := tx.QueryRow(ctx, fmt.Sprintf("SELECT coalesce(max(id), 0) + 1 FROM %s WHERE workspace_id = $1 AND property_id = $2%s", table, where), args...).Scan(&next); err != nil { + return nil, err + } + limit, capped := Caps[kind] + var add []string + var ids []int32 + overflow := false + for _, v := range missing { + if special(v) || !capped || used < limit || elsewhere[v] { + add, ids = append(add, v), append(ids, next) + got[v] = next + next++ + if !special(v) && !elsewhere[v] { + used++ + } + continue + } + overflow = true + } + if _, ok := got[OtherValue]; overflow && !ok { + add, ids = append(add, OtherValue), append(ids, next) + got[OtherValue] = next + } + if len(add) > 0 { + if kind == KindRoute { + inf := make([]bool, len(add)) + for i, v := range add { + inf[i] = inferred[v] + } + _, err = tx.Exec(ctx, `INSERT INTO monitor_routes (workspace_id, property_id, id, pattern, inferred, first_seen_at, last_seen_at) + SELECT $1, $2, u.id, u.pattern, u.inferred, $6, $6 FROM unnest($3::int[], $4::text[], $5::boolean[]) AS u(id, pattern, inferred)`, + ws, prop, ids, add, inf, now) + } else { + _, err = tx.Exec(ctx, `INSERT INTO monitor_names (workspace_id, property_id, kind, id, value, first_seen_at) + SELECT $1, $2, $3, u.id, u.value, $6 FROM unnest($4::int[], $5::text[]) AS u(id, value)`, ws, prop, kind, ids, add, now) + } + if err != nil { + return nil, fmt.Errorf("monitor names: %w", err) + } + } + for _, v := range missing { + if _, ok := got[v]; !ok { + got[v] = got[OtherValue] + } + } + return got, nil +} + +// liveProperties are the write's API properties that still exist, in workspaces that are not +// deleted; rows of others are not written. +func liveProperties(ctx context.Context, tx pgx.Tx, w Write) (map[uuid.UUID]bool, error) { + pairs := map[[2]uuid.UUID]bool{} + for _, r := range w.Minutes { + pairs[[2]uuid.UUID{r.WorkspaceID, r.PropertyID}] = true + } + for _, r := range w.Dims { + pairs[[2]uuid.UUID{r.WorkspaceID, r.PropertyID}] = true + } + for _, c := range w.Counts { + pairs[[2]uuid.UUID{c.WorkspaceID, c.PropertyID}] = true + } + ws, props := make([]uuid.UUID, 0, len(pairs)), make([]uuid.UUID, 0, len(pairs)) + for p := range pairs { + ws, props = append(ws, p[0]), append(props, p[1]) + } + rows, err := tx.Query(ctx, `SELECT p.id FROM analytics_properties p + JOIN unnest($1::uuid[], $2::uuid[]) AS u(workspace_id, id) ON p.workspace_id = u.workspace_id AND p.id = u.id + JOIN workspaces w ON w.id = p.workspace_id + WHERE p.deleted_at IS NULL AND w.deleted_at IS NULL AND p.kind = 'api'`, ws, props) + if err != nil { + return nil, err + } + live, err := pgx.CollectRows(rows, pgx.RowTo[uuid.UUID]) + out := map[uuid.UUID]bool{} + for _, id := range live { + out[id] = true + } + return out, err +} + +// aggColumns are the columns of an Agg, in the order aggArrays fills them. +const aggColumns = "requests, errors, client_errors, sampled, duration_sum, duration_min, duration_max, durations, request_bytes, response_bytes, response_sizes" + +type aggArrays struct { + requests, errors, clientErrors, sampled, durSum, durMin, durMax, reqBytes, respBytes []int64 + durations, sizes [][]byte +} + +func (a *aggArrays) add(g *Agg) { + a.requests, a.errors, a.clientErrors, a.sampled = append(a.requests, g.Requests), append(a.errors, g.Errors), append(a.clientErrors, g.ClientErrors), append(a.sampled, g.Sampled) + a.durSum, a.durMin, a.durMax = append(a.durSum, g.DurationSum), append(a.durMin, g.DurationMin), append(a.durMax, g.DurationMax) + a.reqBytes, a.respBytes = append(a.reqBytes, g.RequestBytes), append(a.respBytes, g.ResponseBytes) + a.durations, a.sizes = append(a.durations, g.Durations.Encode()), append(a.sizes, g.Sizes.Encode()) +} + +func (a *aggArrays) args() []any { + return []any{a.requests, a.errors, a.clientErrors, a.sampled, a.durSum, a.durMin, a.durMax, a.durations, a.reqBytes, a.respBytes, a.sizes} +} + +const aggTypes = "$%d::bigint[], $%d::bigint[], $%d::bigint[], $%d::bigint[], $%d::bigint[], $%d::bigint[], $%d::bigint[], $%d::bytea[], $%d::bigint[], $%d::bigint[], $%d::bytea[]" + +func aggPlaceholders(first int) string { + n := make([]any, 11) + for i := range n { + n[i] = first + i + } + return fmt.Sprintf(aggTypes, n...) +} + +const aggReplace = `requests = EXCLUDED.requests, errors = EXCLUDED.errors, client_errors = EXCLUDED.client_errors, sampled = EXCLUDED.sampled, + duration_sum = EXCLUDED.duration_sum, duration_min = EXCLUDED.duration_min, duration_max = EXCLUDED.duration_max, durations = EXCLUDED.durations, + request_bytes = EXCLUDED.request_bytes, response_bytes = EXCLUDED.response_bytes, response_sizes = EXCLUDED.response_sizes` + +func (s *PGStore) Write(ctx context.Context, w Write, now time.Time) error { + return pgx.BeginFunc(ctx, s.Pool, func(tx pgx.Tx) error { + live, err := liveProperties(ctx, tx, w) + if err != nil { + return err + } + mins := slices.DeleteFunc(slices.Clone(w.Minutes), func(r MinuteRow) bool { return !live[r.PropertyID] }) + slices.SortFunc(mins, func(a, b MinuteRow) int { + return cmp.Or(a.PropertyID.Compare(b.PropertyID), a.Minute.Compare(b.Minute), cmp.Compare(a.Writer, b.Writer), compareCore(a.Key, b.Key)) + }) + if len(mins) > 0 { + n := len(mins) + ws, props, minute, writer := make([]uuid.UUID, n), make([]uuid.UUID, n), make([]time.Time, n), make([]int64, n) + env, svc, route, ver := make([]int32, n), make([]int32, n), make([]int32, n), make([]int32, n) + method, status, source, platform := make([]int16, n), make([]int16, n), make([]int16, n), make([]int16, n) + var aggs aggArrays + for i, r := range mins { + ws[i], props[i], minute[i], writer[i] = r.WorkspaceID, r.PropertyID, r.Minute, r.Writer + k := r.Key + env[i], svc[i], route[i], ver[i] = k.Environment, k.Service, k.Route, k.APIVersion + method[i], status[i], source[i], platform[i] = k.Method, k.Status, int16(k.Source), int16(k.Platform) + aggs.add(&r.Agg) + } + args := append([]any{ws, props, minute, writer, env, svc, route, method, status, ver, source, platform}, aggs.args()...) + if _, err := tx.Exec(ctx, `INSERT INTO monitor_minute (workspace_id, property_id, minute, writer, environment, service, route, method, status, + api_version, source, platform, `+aggColumns+`) +SELECT * FROM unnest($1::uuid[], $2::uuid[], $3::timestamptz[], $4::bigint[], $5::int[], $6::int[], $7::int[], $8::smallint[], $9::smallint[], + $10::int[], $11::smallint[], $12::smallint[], `+aggPlaceholders(13)+`) +ON CONFLICT (property_id, minute, writer, environment, service, route, method, status, api_version, source, platform) DO UPDATE SET `+aggReplace, + args...); err != nil { + return fmt.Errorf("monitor minutes: %w", err) + } + } + dims := slices.DeleteFunc(slices.Clone(w.Dims), func(r DimRow) bool { return !live[r.PropertyID] }) + slices.SortFunc(dims, func(a, b DimRow) int { + return cmp.Or(a.PropertyID.Compare(b.PropertyID), a.Hour.Compare(b.Hour), cmp.Compare(a.Writer, b.Writer), compareDim(a.Key, b.Key)) + }) + if len(dims) > 0 { + n := len(dims) + ws, props, hour, writer := make([]uuid.UUID, n), make([]uuid.UUID, n), make([]time.Time, n), make([]int64, n) + dim, method, class := make([]int16, n), make([]int16, n), make([]int16, n) + value, route := make([]int32, n), make([]int32, n) + var aggs aggArrays + for i, r := range dims { + ws[i], props[i], hour[i], writer[i] = r.WorkspaceID, r.PropertyID, r.Hour, r.Writer + dim[i], value[i], route[i], method[i], class[i] = r.Key.Dim, r.Key.Value, r.Key.Route, r.Key.Method, r.Key.StatusClass + aggs.add(&r.Agg) + } + args := append([]any{ws, props, hour, writer, dim, value, route, method, class}, aggs.args()...) + if _, err := tx.Exec(ctx, `INSERT INTO monitor_dim_hour (workspace_id, property_id, hour, writer, dim, value, route, method, status_class, `+aggColumns+`) +SELECT * FROM unnest($1::uuid[], $2::uuid[], $3::timestamptz[], $4::bigint[], $5::smallint[], $6::int[], $7::int[], $8::smallint[], $9::smallint[], + `+aggPlaceholders(10)+`) +ON CONFLICT (property_id, hour, writer, dim, value, route, method, status_class) DO UPDATE SET `+aggReplace, args...); err != nil { + return fmt.Errorf("monitor dimensions: %w", err) + } + } + if err := markHours(ctx, tx, w.Hours, live); err != nil { + return err + } + if err := addCounts(ctx, tx, w.Counts, live); err != nil { + return err + } + if err := addUsage(ctx, tx, monthOf(now), w.Usage); err != nil { + return err + } + return touchRoutes(ctx, tx, mins, now) + }) +} + +func compareCore(a, b CoreKey) int { + return cmp.Or(cmp.Compare(a.Environment, b.Environment), cmp.Compare(a.Service, b.Service), cmp.Compare(a.Route, b.Route), + cmp.Compare(a.Method, b.Method), cmp.Compare(a.Status, b.Status), cmp.Compare(a.APIVersion, b.APIVersion), + cmp.Compare(a.Source, b.Source), cmp.Compare(a.Platform, b.Platform)) +} + +func compareDim(a, b DimKey) int { + return cmp.Or(cmp.Compare(a.Dim, b.Dim), cmp.Compare(a.Value, b.Value), cmp.Compare(a.Route, b.Route), cmp.Compare(a.Method, b.Method), + cmp.Compare(a.StatusClass, b.StatusClass)) +} + +// markHours records the hours that have rows, for the folds, and marks those already folded dirty. +func markHours(ctx context.Context, tx pgx.Tx, hours []HourMark, live map[uuid.UUID]bool) error { + hours = slices.DeleteFunc(slices.Clone(hours), func(h HourMark) bool { return !live[h.PropertyID] }) + if len(hours) == 0 { + return nil + } + n := len(hours) + ws, props, hour, day, capped := make([]uuid.UUID, n), make([]uuid.UUID, n), make([]time.Time, n), make([]time.Time, n), make([]bool, n) + for i, h := range hours { + ws[i], props[i], hour[i], day[i], capped[i] = h.WorkspaceID, h.PropertyID, h.Hour, h.Day, h.Capped + } + if _, err := tx.Exec(ctx, ` +INSERT INTO monitor_rollup_hours (workspace_id, property_id, hour, day, capped) +SELECT * FROM unnest($1::uuid[], $2::uuid[], $3::timestamptz[], $4::date[], $5::boolean[]) +ON CONFLICT (property_id, hour) DO UPDATE SET + dirty = monitor_rollup_hours.dirty OR monitor_rollup_hours.built_at IS NOT NULL, + capped = monitor_rollup_hours.capped OR EXCLUDED.capped`, ws, props, hour, day, capped); err != nil { + return fmt.Errorf("monitor hours: %w", err) + } + return nil +} + +func addCounts(ctx context.Context, tx pgx.Tx, counts []Count, live map[uuid.UUID]bool) error { + counts = slices.DeleteFunc(slices.Clone(counts), func(c Count) bool { return !live[c.PropertyID] || c.N == 0 }) + if len(counts) == 0 { + return nil + } + slices.SortFunc(counts, func(a, b Count) int { + return cmp.Or(a.PropertyID.Compare(b.PropertyID), a.Day.Compare(b.Day), strings.Compare(a.Reason, b.Reason)) + }) + n := len(counts) + ws, props, days, reasons, records := make([]uuid.UUID, n), make([]uuid.UUID, n), make([]time.Time, n), make([]string, n), make([]int64, n) + for i, c := range counts { + ws[i], props[i], days[i], reasons[i], records[i] = c.WorkspaceID, c.PropertyID, c.Day, c.Reason, c.N + } + if _, err := tx.Exec(ctx, ` +INSERT INTO monitor_ingest_stats (workspace_id, property_id, day, reason, records) +SELECT u.workspace_id, u.property_id, u.day, u.reason, sum(u.records) +FROM unnest($1::uuid[], $2::uuid[], $3::date[], $4::text[], $5::bigint[]) AS u(workspace_id, property_id, day, reason, records) +GROUP BY u.workspace_id, u.property_id, u.day, u.reason +ORDER BY u.property_id, u.day, u.reason +ON CONFLICT (property_id, day, reason) DO UPDATE SET records = monitor_ingest_stats.records + EXCLUDED.records`, + ws, props, days, reasons, records); err != nil { + return fmt.Errorf("monitor ingest stats: %w", err) + } + return nil +} + +func addUsage(ctx context.Context, tx pgx.Tx, month time.Time, usage map[uuid.UUID]int64) error { + for _, ws := range slices.SortedFunc(maps.Keys(usage), func(a, b uuid.UUID) int { return a.Compare(b) }) { + if usage[ws] == 0 { + continue + } + if _, err := tx.Exec(ctx, ` +INSERT INTO usage_counters (workspace_id, month, monitor_requests) SELECT $1, $2, $3 WHERE EXISTS (SELECT 1 FROM workspaces WHERE id = $1) +ON CONFLICT (workspace_id, month) DO UPDATE SET monitor_requests = usage_counters.monitor_requests + EXCLUDED.monitor_requests`, + ws, month, usage[ws]); err != nil { + return fmt.Errorf("monitor usage: %w", err) + } + } + return nil +} + +func touchRoutes(ctx context.Context, tx pgx.Tx, mins []MinuteRow, now time.Time) error { + seen := map[[2]uuid.UUID]map[int32]bool{} + for _, r := range mins { + k := [2]uuid.UUID{r.WorkspaceID, r.PropertyID} + if seen[k] == nil { + seen[k] = map[int32]bool{} + } + seen[k][r.Key.Route] = true + } + for _, k := range slices.SortedFunc(maps.Keys(seen), func(a, b [2]uuid.UUID) int { return a[1].Compare(b[1]) }) { + ids := slices.Sorted(maps.Keys(seen[k])) + if _, err := tx.Exec(ctx, `UPDATE monitor_routes SET last_seen_at = $4 + WHERE workspace_id = $1 AND property_id = $2 AND id = ANY($3) AND last_seen_at < $5`, k[0], k[1], ids, now, now.Add(-time.Minute)); err != nil { + return fmt.Errorf("monitor routes: %w", err) + } + } + return nil +} diff --git a/api/internal/monitor/query.go b/api/internal/monitor/query.go new file mode 100644 index 00000000..c11e804f --- /dev/null +++ b/api/internal/monitor/query.go @@ -0,0 +1,722 @@ +package monitor + +import ( + "context" + "errors" + "fmt" + "slices" + "strconv" + "strings" + "time" + "uuid" + + "github.com/jackc/pgx/v5" + + "github.com/productdevbook/yuva/api/internal/analytics" +) + +const ( + ServedRollups = analytics.ServedRollups + ServedRaw = analytics.ServedRaw + ServedMixed = analytics.ServedMixed + + MaxMinuteBuckets = 1440 + minuteLabel = "2006-01-02T15:04" +) + +// Dims are the dimensions an API query breaks down and filters by; Metrics its metrics. +var ( + Dims = []string{"route", "method", "status", "status_class", "environment"} + Metrics = []string{"requests", "requests_per_minute", "errors", "error_rate", "client_errors", "client_error_rate", "p50", "p95", "p99", "avg_duration"} + Grains = []string{"time:minute", "time:hour", "time:day"} +) + +var ErrBadQuery = analytics.ErrBadQuery + +// ErrMinuteRange: a time:minute query reaches past the minute rows or has too many buckets. +var ErrMinuteRange = errors.New("time:minute serves at most 1,440 minutes within the minute rows") + +func badQuery(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrBadQuery, fmt.Sprintf(format, args...)) +} + +type Filter = analytics.Filter + +// Query asks for an API property's requests of [Start, End), broken down by up to two dimensions +// (one may be a time grain) and narrowed by filters. MinuteFrom is the first minute minute rows are +// kept for; Days says the range is whole local days, which daily rows may answer. +type Query struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Location *time.Location + Start, End time.Time + Days bool + Metrics []string + Dimensions []string + Filters []Filter + MinuteFrom time.Time + Now time.Time +} + +type Row struct { + Dims []string + Agg Agg + // Minutes is how many minutes of the row's range have passed, for requests_per_minute. + Minutes float64 +} + +type Result struct { + Rows []Row + Totals Agg + Minutes float64 + ServedFrom string + Caps []string +} + +// Metric is a metric of a's requests over minutes. +func (a *Agg) Metric(name string, minutes float64) float64 { + pct := func(n int64) float64 { + if a.Requests == 0 { + return 0 + } + return round(float64(n)*100/float64(a.Requests), 2) + } + switch name { + case "requests": + return float64(a.Requests) + case "requests_per_minute": + if minutes <= 0 { + return 0 + } + return round(float64(a.Requests)/minutes, 2) + case "errors": + return float64(a.Errors) + case "error_rate": + return pct(a.Errors) + case "client_errors": + return float64(a.ClientErrors) + case "client_error_rate": + return pct(a.ClientErrors) + case "p50": + return round(a.DurationQuantile(0.5)/1000, 6) + case "p95": + return round(a.DurationQuantile(0.95)/1000, 6) + case "p99": + return round(a.DurationQuantile(0.99)/1000, 6) + case "avg_duration": + if a.Requests == 0 { + return 0 + } + return round(float64(a.DurationSum)/float64(a.Requests)/1000, 6) + } + return 0 +} + +func round(v float64, digits int) float64 { + p, err := strconv.ParseFloat(strconv.FormatFloat(v, 'f', digits, 64), 64) + if err != nil { + return v + } + return p +} + +type plan struct { + q Query + grain string + dims []string + days []time.Time + hours []time.Time + minutes [][2]time.Time + sketches bool +} + +func (p *plan) parse() error { + q := p.q + if !q.End.After(q.Start) { + return badQuery("the range ends before it starts") + } + if len(q.Dimensions) > 2 { + return badQuery("at most two dimensions") + } + for _, d := range q.Dimensions { + switch { + case slices.Contains(Grains, d): + if p.grain != "" { + return badQuery("a query has at most one time grain") + } + p.grain = strings.TrimPrefix(d, "time:") + case slices.Contains(Dims, d): + if slices.Contains(p.dims, d) { + return badQuery("dimension %q is listed twice", d) + } + p.dims = append(p.dims, d) + default: + return badQuery("the api dataset breaks down by %s and time:minute, time:hour or time:day, not %q", strings.Join(Dims, ", "), d) + } + } + for _, m := range q.Metrics { + if !slices.Contains(Metrics, m) { + return badQuery("the api dataset offers %s, not %s", strings.Join(Metrics, ", "), m) + } + if m == "p50" || m == "p95" || m == "p99" { + p.sketches = true + } + } + for _, f := range q.Filters { + if !slices.Contains(Dims, f.Dimension) { + return badQuery("the api dataset filters on %s, not %q", strings.Join(Dims, ", "), f.Dimension) + } + if !slices.Contains([]string{"is", "is_not", "contains", "not_contains", "matches"}, f.Op) || len(f.Values) == 0 { + return badQuery("the filter on %q needs an op of is, is_not, contains, not_contains or matches and at least one value", f.Dimension) + } + } + if p.grain == "minute" && (q.Start.Before(q.MinuteFrom) || q.End.Sub(q.Start) > MaxMinuteBuckets*time.Minute) { + return ErrMinuteRange + } + return nil +} + +type hourState struct { + hour, day time.Time + built, dayBuilt bool +} + +// tiers decides where each part of the range is read from: closed days from daily rows, closed +// hours from hourly rows, everything else from minute rows. Hours without a row had no requests. +func (p *plan) tiers(ctx context.Context, tx pgx.Tx) error { + q := p.q + if p.grain == "minute" { + p.minutes = [][2]time.Time{{q.Start, q.End}} + return nil + } + rows, err := tx.Query(ctx, `SELECT hour, day, built_at IS NOT NULL AND NOT dirty, day_built_at IS NOT NULL FROM monitor_rollup_hours + WHERE workspace_id = $1 AND property_id = $2 AND hour > $3 AND hour < $4 ORDER BY hour`, q.WorkspaceID, q.PropertyID, q.Start.Add(-time.Hour), q.End) + if err != nil { + return err + } + var states []hourState + for rows.Next() { + var h hourState + if err := rows.Scan(&h.hour, &h.day, &h.built, &h.dayBuilt); err != nil { + rows.Close() + return err + } + h.hour, h.day = h.hour.UTC(), h.day.UTC() + states = append(states, h) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + byDay := map[time.Time]bool{} + if q.Days && p.grain != "hour" { + ready := map[time.Time]bool{} + for _, h := range states { + if _, ok := ready[h.day]; !ok { + ready[h.day] = true + } + ready[h.day] = ready[h.day] && h.built && h.dayBuilt + } + for d, ok := range ready { + if ok && !DayStart(d, q.Location).Before(q.Start) && !DayStart(d.AddDate(0, 0, 1), q.Location).After(q.End) { + byDay[d] = true + p.days = append(p.days, d) + } + } + slices.SortFunc(p.days, func(a, b time.Time) int { return a.Compare(b) }) + } + for _, h := range states { + if byDay[h.day] { + continue + } + end := h.hour.Add(time.Hour) + if h.built && !h.hour.Before(q.Start) && !end.After(q.End) { + p.hours = append(p.hours, h.hour) + continue + } + from, to := maxTime(h.hour, q.Start), minTime(end, q.End) + if n := len(p.minutes); n > 0 && p.minutes[n-1][1].Equal(from) { + p.minutes[n-1][1] = to + } else { + p.minutes = append(p.minutes, [2]time.Time{from, to}) + } + } + return nil +} + +// names maps a property's dictionary ids to their values. +type nameMaps struct { + routes, envs map[int32]string +} + +func loadNames(ctx context.Context, tx pgx.Tx, ws, prop uuid.UUID) (nameMaps, error) { + out := nameMaps{routes: map[int32]string{}, envs: map[int32]string{}} + rows, err := tx.Query(ctx, `SELECT id, pattern FROM monitor_routes WHERE workspace_id = $1 AND property_id = $2`, ws, prop) + if err != nil { + return out, err + } + if err := collectNames(rows, out.routes); err != nil { + return out, err + } + rows, err = tx.Query(ctx, `SELECT id, value FROM monitor_names WHERE workspace_id = $1 AND property_id = $2 AND kind = 'environment'`, ws, prop) + if err != nil { + return out, err + } + return out, collectNames(rows, out.envs) +} + +func collectNames(rows pgx.Rows, into map[int32]string) error { + defer rows.Close() + for rows.Next() { + var id int32 + var v string + if err := rows.Scan(&id, &v); err != nil { + return err + } + into[id] = v + } + return rows.Err() +} + +// matches is a filter's test of one value, as the analytics filters compare: whole values, parts +// without regard to case, or patterns (on routes * is a segment and ** any number of them). +func matches(f Filter, v string) bool { + for _, want := range f.Values { + switch f.Op { + case "is", "is_not": + if v == want { + return true + } + case "contains", "not_contains": + if strings.Contains(strings.ToLower(v), strings.ToLower(want)) { + return true + } + case "matches": + if f.Dimension == "route" { + if analytics.MatchPath(want, v) { + return true + } + } else if wildcard(want, v) { + return true + } + } + } + return false +} + +func wildcard(pattern, v string) bool { + parts := strings.Split(pattern, "*") + if len(parts) == 1 { + return pattern == v + } + if !strings.HasPrefix(v, parts[0]) { + return false + } + v = v[len(parts[0]):] + for _, part := range parts[1 : len(parts)-1] { + i := strings.Index(v, part) + if i < 0 { + return false + } + v = v[i+len(part):] + } + return strings.HasSuffix(v, parts[len(parts)-1]) +} + +func negative(op string) bool { return op == "is_not" || op == "not_contains" } + +// filterSQL turns the filters into conditions on ids and codes, numbering parameters from next. +func filterSQL(fs []Filter, names nameMaps, next int) (string, []any) { + var conds []string + var args []any + for _, f := range fs { + var col string + set := []int32{} + switch f.Dimension { + case "route": + col = "route" + for id, v := range names.routes { + if matches(f, v) { + set = append(set, id) + } + } + case "environment": + col = "environment" + for id, v := range names.envs { + if matches(f, v) { + set = append(set, id) + } + } + case "method": + col = "method" + for code := range int16(len(methods)) { + if matches(f, MethodName(code)) { + set = append(set, int32(code)) + } + } + case "status": + col = "status" + for code := range int32(600) { + if matches(f, strconv.Itoa(int(code))) { + set = append(set, code) + } + } + case "status_class": + col = "(status / 100)" + for class := range int16(6) { + if matches(f, StatusClassName(class)) { + set = append(set, int32(class)) + } + } + } + slices.Sort(set) + cond := fmt.Sprintf("%s = ANY($%d::int[])", col, next+len(args)) + if negative(f.Op) { + cond = "NOT " + cond + } + conds, args = append(conds, cond), append(args, set) + } + if len(conds) == 0 { + return "", nil + } + return " AND " + strings.Join(conds, " AND "), args +} + +type tier struct { + table, timeCol, where string + args []any +} + +// bucketSQL is a tier's expression for the query's grain: instants for minutes and hours, local +// dates for days. +func (p *plan) bucketSQL(t tier, tz int) string { + switch p.grain { + case "minute": + return t.timeCol + case "hour": + if t.timeCol == "minute" { + return fmt.Sprintf("(minute - make_interval(mins => extract(minute FROM minute AT TIME ZONE $%d)::int))", tz) + } + return t.timeCol + case "day": + if t.timeCol == "day" { + return "day::timestamp AT TIME ZONE 'UTC'" + } + return fmt.Sprintf("((%s AT TIME ZONE $%d)::date)::timestamp AT TIME ZONE 'UTC'", t.timeCol, tz) + } + return "" +} + +func dimSQL(d string) string { + switch d { + case "status_class": + return "(status / 100)::int" + case "method", "status": + return d + "::int" + } + return d +} + +type group struct { + bucket time.Time + vals []int32 + agg Agg +} + +func (s *PGStore) Query(ctx context.Context, q Query) (Result, error) { + p := &plan{q: q} + if err := p.parse(); err != nil { + return Result{}, err + } + var res Result + err := pgx.BeginTxFunc(ctx, s.Pool, pgx.TxOptions{AccessMode: pgx.ReadOnly}, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, "SET LOCAL statement_timeout = "+strconv.Itoa(int(analytics.QueryTimeout.Milliseconds()))); err != nil { + return err + } + if err := p.tiers(ctx, tx); err != nil { + return err + } + names, err := loadNames(ctx, tx, q.WorkspaceID, q.PropertyID) + if err != nil { + return err + } + var tiers []tier + if len(p.days) > 0 { + tiers = append(tiers, tier{table: "monitor_day", timeCol: "day", where: "day = ANY($3::date[])", args: []any{p.days}}) + } + if len(p.hours) > 0 { + tiers = append(tiers, tier{table: "monitor_hour", timeCol: "hour", where: "hour = ANY($3::timestamptz[])", args: []any{p.hours}}) + } + if len(p.minutes) > 0 { + var or []string + var args []any + for _, r := range p.minutes { + or = append(or, fmt.Sprintf("(minute >= $%d AND minute < $%d)", 3+len(args), 4+len(args))) + args = append(args, r[0], r[1]) + } + tiers = append(tiers, tier{table: MinuteTable, timeCol: "minute", where: "(" + strings.Join(or, " OR ") + ")", args: args}) + } + groups := map[string]*group{} + var order []string + for _, t := range tiers { + args := append([]any{q.WorkspaceID, q.PropertyID}, t.args...) + var cols []string + tz := 0 + if p.grain != "" { + if p.grain != "minute" && !(p.grain == "hour" && t.timeCol == "hour") && !(p.grain == "day" && t.timeCol == "day") { + args = append(args, q.Location.String()) + tz = len(args) + } + cols = append(cols, p.bucketSQL(t, tz)) + } + for _, d := range p.dims { + cols = append(cols, dimSQL(d)) + } + fsql, fargs := filterSQL(q.Filters, names, len(args)+1) + args = append(args, fargs...) + sel := `sum(requests)::bigint, sum(errors)::bigint, sum(client_errors)::bigint, sum(sampled)::bigint, sum(duration_sum)::bigint, + min(duration_min)::bigint, max(duration_max)::bigint, sum(request_bytes)::bigint, sum(response_bytes)::bigint` + if p.sketches { + sel += ", array_agg(durations)" + } + groupBy := "" + if len(cols) > 0 { + pos := make([]string, len(cols)) + for i := range cols { + pos[i] = strconv.Itoa(i + 1) + } + groupBy = " GROUP BY " + strings.Join(pos, ", ") + sel = strings.Join(cols, ", ") + ", " + sel + } + rows, err := tx.Query(ctx, "SELECT "+sel+" FROM "+t.table+" WHERE workspace_id = $1 AND property_id = $2 AND "+t.where+fsql+groupBy, args...) + if err != nil { + return fmt.Errorf("monitor query: %w", err) + } + for rows.Next() { + var g group + g.vals = make([]int32, len(p.dims)) + var dest []any + if p.grain != "" { + dest = append(dest, &g.bucket) + } + for i := range g.vals { + dest = append(dest, &g.vals[i]) + } + a := &g.agg + dest = append(dest, &a.Requests, &a.Errors, &a.ClientErrors, &a.Sampled, &a.DurationSum, &a.DurationMin, &a.DurationMax, + &a.RequestBytes, &a.ResponseBytes) + var sketches [][]byte + if p.sketches { + dest = append(dest, &sketches) + } + if err := rows.Scan(dest...); err != nil { + rows.Close() + return err + } + a.Durations, a.Sizes = NewSketch(DurationScale), NewSketch(SizeScale) + for _, raw := range sketches { + sk, err := DecodeSketch(raw) + if err != nil { + rows.Close() + return err + } + a.Durations.Merge(sk) + } + key := g.bucket.UTC().Format(time.RFC3339) + "|" + fmt.Sprint(g.vals) + if cur := groups[key]; cur != nil { + cur.agg.Merge(&g.agg) + } else { + g.bucket = g.bucket.UTC() + groups[key] = &g + order = append(order, key) + } + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("monitor query: %w", err) + } + } + res = p.result(groups, order, names) + res.Caps, err = caps(ctx, tx, q) + return err + }) + if err != nil { + return Result{}, err + } + switch { + case len(p.minutes) == 0: + res.ServedFrom = ServedRollups + case len(p.days) == 0 && len(p.hours) == 0: + res.ServedFrom = ServedRaw + default: + res.ServedFrom = ServedMixed + } + return res, nil +} + +// passed is how many minutes of [from, to) are not in the future. +func passed(from, to, now time.Time) float64 { + end := minTime(to, now.Truncate(time.Minute).Add(time.Minute)) + if !end.After(from) { + return 0 + } + return end.Sub(from).Minutes() +} + +func (p *plan) step(t time.Time) time.Time { + switch p.grain { + case "minute": + return t.Add(time.Minute) + case "hour": + return t.Add(time.Hour) + } + return DayStart(LocalDay(t, p.q.Location).AddDate(0, 0, 1), p.q.Location) +} + +// bucketStart is a bucket's first instant: day buckets are labelled with their local date. +func (p *plan) bucketStart(b time.Time) time.Time { + if p.grain == "day" { + return DayStart(b, p.q.Location) + } + return b +} + +func (p *plan) label(b time.Time) string { + if p.grain == "day" { + return b.Format(time.DateOnly) + } + return b.In(p.q.Location).Format(minuteLabel) +} + +func (p *plan) result(groups map[string]*group, order []string, names nameMaps) Result { + q := p.q + res := Result{Totals: NewAgg(), Minutes: passed(q.Start, q.End, q.Now)} + if p.grain != "" && len(p.dims) == 0 { + have := map[time.Time]bool{} + for _, g := range groups { + have[p.bucketStart(g.bucket)] = true + } + for t := p.firstBucket(); t.Before(q.End); t = p.step(t) { + if !have[t] { + b := t + if p.grain == "day" { + b = LocalDay(t, q.Location) + } + key := b.Format(time.RFC3339) + "|[]" + groups[key] = &group{bucket: b, agg: NewAgg()} + order = append(order, key) + } + } + } + for _, key := range order { + g := groups[key] + res.Totals.Merge(&g.agg) + var dims []string + minutes := res.Minutes + if p.grain != "" { + start := p.bucketStart(g.bucket) + dims = append(dims, p.label(g.bucket)) + minutes = passed(maxTime(start, q.Start), minTime(p.step(start), q.End), q.Now) + } + for i, d := range p.dims { + v := g.vals[i] + switch d { + case "route": + dims = append(dims, names.routes[v]) + case "environment": + dims = append(dims, names.envs[v]) + case "method": + dims = append(dims, MethodName(int16(v))) + case "status": + dims = append(dims, strconv.Itoa(int(v))) + case "status_class": + dims = append(dims, StatusClassName(int16(v))) + } + } + if p.grain != "" || len(p.dims) > 0 { + res.Rows = append(res.Rows, Row{Dims: dims, Agg: g.agg, Minutes: minutes}) + } + } + if p.grain != "" { + slices.SortStableFunc(res.Rows, func(a, b Row) int { return strings.Compare(a.Dims[0], b.Dims[0]) }) + } + return res +} + +func (p *plan) firstBucket() time.Time { + q := p.q + switch p.grain { + case "minute": + return q.Start.Truncate(time.Minute) + case "hour": + return LocalHour(q.Start, q.Location) + } + return DayStart(LocalDay(q.Start, q.Location), q.Location) +} + +// caps lists the caps the property has reached, and dimension_rows when an hour of the range did. +func caps(ctx context.Context, tx pgx.Tx, q Query) ([]string, error) { + out := []string{} + var routes int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM monitor_routes WHERE workspace_id = $1 AND property_id = $2 AND pattern NOT IN ('(other)', '(unmatched)')`, + q.WorkspaceID, q.PropertyID).Scan(&routes); err != nil { + return nil, err + } + if routes >= Caps[KindRoute] { + out = append(out, "routes") + } + rows, err := tx.Query(ctx, `SELECT kind, count(*) FROM monitor_names WHERE workspace_id = $1 AND property_id = $2 AND value <> '(other)' + AND kind IN ('consumer', 'api_version', 'app_version', 'environment') GROUP BY kind`, q.WorkspaceID, q.PropertyID) + if err != nil { + return nil, err + } + counts := map[string]int{} + for rows.Next() { + var kind string + var n int + if err := rows.Scan(&kind, &n); err != nil { + rows.Close() + return nil, err + } + counts[kind] = n + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + var services int + if err := tx.QueryRow(ctx, `SELECT count(DISTINCT value) FROM monitor_names WHERE workspace_id = $1 AND kind = 'service' AND value <> '(other)'`, + q.WorkspaceID).Scan(&services); err != nil { + return nil, err + } + counts[KindService] = services + for _, k := range []struct{ kind, name string }{{KindConsumer, "consumers"}, {KindAPIVersion, "api_versions"}, {KindAppVersion, "app_versions"}, + {KindEnvironment, "environments"}, {KindService, "services"}} { + if counts[k.kind] >= Caps[k.kind] { + out = append(out, k.name) + } + } + var capped bool + if err := tx.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM monitor_rollup_hours WHERE workspace_id = $1 AND property_id = $2 AND capped + AND hour > $3 AND hour < $4)`, q.WorkspaceID, q.PropertyID, q.Start.Add(-time.Hour), q.End).Scan(&capped); err != nil { + return nil, err + } + if capped { + out = append(out, "dimension_rows") + } + return out, nil +} + +func minTime(a, b time.Time) time.Time { + if b.Before(a) { + return b + } + return a +} + +func maxTime(a, b time.Time) time.Time { + if b.After(a) { + return b + } + return a +} diff --git a/api/internal/monitor/route.go b/api/internal/monitor/route.go new file mode 100644 index 00000000..a5aba648 --- /dev/null +++ b/api/internal/monitor/route.go @@ -0,0 +1,139 @@ +package monitor + +import ( + "regexp" + "strconv" + "strings" +) + +const ( + // OtherValue is what a value past its cap is counted as; Unmatched the route of a 404 or 405 + // that named no pattern; Scrubbed an e-mail-shaped consumer. + OtherValue = "(other)" + Unmatched = "(unmatched)" + Scrubbed = "(scrubbed)" + IDSegment = "{id}" + MaxRoute = 300 + maxSegment = 40 + minTokenLength = 16 +) + +var methods = []string{"_OTHER", "GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"} + +// Method is a request method's code: the RFC 9110 methods and PATCH, else 0 (_OTHER). +func Method(m string) int16 { + m = strings.ToUpper(strings.TrimSpace(m)) + for i, name := range methods[1:] { + if m == name { + return int16(i + 1) + } + } + return 0 +} + +func MethodName(code int16) string { + if code < 0 || int(code) >= len(methods) { + return methods[0] + } + return methods[code] +} + +// StatusClass is a status's hundreds, 0 for no response. +func StatusClass(status int16) int16 { return status / 100 } + +func StatusClassName(class int16) string { + if class <= 0 { + return "none" + } + return strconv.Itoa(int(class)) + "xx" +} + +// IsError: a 5xx, or no response at all. IsClientError: a 4xx. +func IsError(status int16) bool { return status == 0 || (status >= 500 && status <= 599) } +func IsClientError(status int16) bool { return status >= 400 && status <= 499 } + +var ( + uuidSegment = regexp.MustCompile(`^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}$`) + dateSegment = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$`) + hexSegment = regexp.MustCompile(`^[0-9a-fA-F]+$`) + tokenSegment = regexp.MustCompile(`^[A-Za-z0-9_-]+={0,2}$`) + emailSegment = regexp.MustCompile(`^[^@\s]+(@|%40)[^@\s]+\.[^@\s.]+$`) +) + +func isNumber(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// idLike: a segment that names one record rather than a part of the API. A token of 16 or more +// letters, digits, - and _ counts only with two digits or more, or with both cases and a digit, +// so words such as user-preferences stay part of the route. +func idLike(seg string) bool { + switch { + case seg == "": + return false + case len(seg) > maxSegment, isNumber(seg), uuidSegment.MatchString(seg), dateSegment.MatchString(seg), emailSegment.MatchString(seg): + return true + case len(seg) < minTokenLength: + return false + case hexSegment.MatchString(seg): + return true + case !tokenSegment.MatchString(seg): + return false + } + digits, upper, lower := 0, false, false + for _, r := range seg { + switch { + case r >= '0' && r <= '9': + digits++ + case r >= 'A' && r <= 'Z': + upper = true + case r >= 'a' && r <= 'z': + lower = true + } + } + return digits >= 2 || (digits == 1 && upper && lower) +} + +// patternSegment: a router's placeholder, kept as it is. +func patternSegment(seg string) bool { + return strings.HasPrefix(seg, "{") || strings.HasPrefix(seg, ":") || strings.HasPrefix(seg, "*") || strings.HasPrefix(seg, "[") || + strings.HasPrefix(seg, "<") +} + +// InferRoute turns a path into a pattern: the query and fragment are cut and every segment that +// looks like an id becomes {id}. It also cleans a router's pattern, so an id put into a route by +// mistake never becomes a route of its own. +func InferRoute(path string) string { + if i := strings.IndexAny(path, "?#"); i >= 0 { + path = path[:i] + } + if path == "" || path[0] != '/' { + path = "/" + path + } + segs := strings.Split(path[1:], "/") + for i, seg := range segs { + if !patternSegment(seg) && idLike(seg) { + segs[i] = IDSegment + } + } + return "/" + strings.Join(segs, "/") +} + +var emailShape = regexp.MustCompile(`^[^@\s]+@[^@\s]+\.[^@\s.]+$`) + +// ScrubConsumer keeps a consumer's name unless it is shaped like an e-mail address: a consumer is +// an account or an API client, never a person's address. +func ScrubConsumer(v string) string { + if strings.Contains(v, "@") && emailShape.MatchString(v) { + return Scrubbed + } + return v +} diff --git a/api/internal/monitor/route_test.go b/api/internal/monitor/route_test.go new file mode 100644 index 00000000..f292fd54 --- /dev/null +++ b/api/internal/monitor/route_test.go @@ -0,0 +1,48 @@ +package monitor + +import "testing" + +func TestInferRoute(t *testing.T) { + for _, c := range []struct{ path, want string }{ + {"/users/42", "/users/{id}"}, + {"/users/42/orders/7?expand=items#top", "/users/{id}/orders/{id}"}, + {"/v1/accounts/0b6a3e9a-2c55-4f7e-9c4f-58f1a5d2a0b1", "/v1/accounts/{id}"}, + {"/v1/accounts/0b6a3e9a2c554f7e9c4f58f1a5d2a0b1/keys", "/v1/accounts/{id}/keys"}, + {"/reports/2026-10-12", "/reports/{id}"}, + {"/blobs/deadbeefcafebabe1234", "/blobs/{id}"}, + {"/tokens/01ARZ3NDEKTSV4RRFFQ69G5FAV", "/tokens/{id}"}, + {"/customers/cus_NffrFeUfNV2Hib", "/customers/{id}"}, + {"/people/ayse@example.com", "/people/{id}"}, + {"/people/ayse%40example.com/notes", "/people/{id}/notes"}, + {"/search/" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnop", "/search/{id}"}, + {"/user-preferences/notification-settings", "/user-preferences/notification-settings"}, + {"/oauth2-callback-handler", "/oauth2-callback-handler"}, + {"/v2/items", "/v2/items"}, + {"/users/{id}", "/users/{id}"}, + {"/users/:id/posts/*", "/users/:id/posts/*"}, + {"/", "/"}, + {"", "/"}, + {"users/42", "/users/{id}"}, + } { + if got := InferRoute(c.path); got != c.want { + t.Errorf("InferRoute(%q) = %q, want %q", c.path, got, c.want) + } + } +} + +func TestMethodsAndStatus(t *testing.T) { + for _, c := range []struct { + in string + want string + }{{"get", "GET"}, {"PATCH", "PATCH"}, {"PROPFIND", "_OTHER"}, {"", "_OTHER"}, {"QUERY", "_OTHER"}} { + if got := MethodName(Method(c.in)); got != c.want { + t.Errorf("method %q = %q, want %q", c.in, got, c.want) + } + } + if !IsError(0) || !IsError(503) || IsError(404) || !IsClientError(404) || IsClientError(500) || IsClientError(399) { + t.Fatal("error classes") + } + if StatusClassName(StatusClass(0)) != "none" || StatusClassName(StatusClass(204)) != "2xx" || StatusClassName(StatusClass(599)) != "5xx" { + t.Fatal("status classes") + } +} diff --git a/api/internal/monitor/sketch.go b/api/internal/monitor/sketch.go new file mode 100644 index 00000000..a771da65 --- /dev/null +++ b/api/internal/monitor/sketch.go @@ -0,0 +1,223 @@ +package monitor + +import ( + "encoding/binary" + "errors" + "math" + "slices" +) + +const ( + // DurationScale gives buckets of 2^(1/32), so a duration percentile is within 1.1% of the + // exact value; SizeScale buckets of 2^(1/4), within 9%. Both are the bucket layouts of an + // OpenTelemetry exponential histogram at that scale. + DurationScale = 5 + SizeScale = 2 +) + +type bucket struct { + index int32 + count uint64 +} + +// Sketch is a sparse log-bucketed histogram: bucket i holds the values in (base^i, base^(i+1)] +// with base 2^(2^-scale), and zero holds the zeros. Adding counts per bucket makes merging exact +// and associative. +type Sketch struct { + scale int8 + zero uint64 + buckets []bucket +} + +func NewSketch(scale int8) Sketch { return Sketch{scale: scale} } + +func (s Sketch) Scale() int8 { return s.scale } + +func (s Sketch) base() float64 { return math.Exp2(math.Exp2(-float64(s.scale))) } + +func (s Sketch) lower(i int32) float64 { return math.Exp2(float64(i) * math.Exp2(-float64(s.scale))) } + +func (s Sketch) index(v float64) int32 { + i := int32(math.Ceil(math.Log2(v)*math.Exp2(float64(s.scale)))) - 1 + for v <= s.lower(i) { + i-- + } + for v > s.lower(i+1) { + i++ + } + return i +} + +// Add counts n values of v; values at or below zero count as zero. +func (s *Sketch) Add(v int64, n uint64) { + if n == 0 { + return + } + if v <= 0 { + s.zero += n + return + } + i := s.index(float64(v)) + at, found := slices.BinarySearchFunc(s.buckets, i, func(b bucket, i int32) int { return int(b.index - i) }) + if found { + s.buckets[at].count += n + return + } + s.buckets = slices.Insert(s.buckets, at, bucket{index: i, count: n}) +} + +// Count is how many values the sketch holds. +func (s Sketch) Count() uint64 { + n := s.zero + for _, b := range s.buckets { + n += b.count + } + return n +} + +// Merge adds o's counts; sketches of different scales are not merged. +func (s *Sketch) Merge(o Sketch) { + if o.zero == 0 && len(o.buckets) == 0 { + return + } + if s.zero == 0 && len(s.buckets) == 0 { + s.scale = o.scale + } + if s.scale != o.scale { + return + } + s.zero += o.zero + if len(s.buckets) == 0 { + s.buckets = slices.Clone(o.buckets) + return + } + out := make([]bucket, 0, len(s.buckets)+len(o.buckets)) + a, b := s.buckets, o.buckets + for len(a) > 0 && len(b) > 0 { + switch { + case a[0].index < b[0].index: + out, a = append(out, a[0]), a[1:] + case a[0].index > b[0].index: + out, b = append(out, b[0]), b[1:] + default: + out, a, b = append(out, bucket{a[0].index, a[0].count + b[0].count}), a[1:], b[1:] + } + } + s.buckets = append(append(out, a...), b...) +} + +func (s Sketch) Clone() Sketch { + return Sketch{scale: s.scale, zero: s.zero, buckets: slices.Clone(s.buckets)} +} + +// value is the bucket's point closest, relative to the value, to every value in it: the harmonic +// mean of its bounds, at most (base-1)/(base+1) away. +func (s Sketch) value(i int32) float64 { + b := s.base() + return s.lower(i) * 2 * b / (1 + b) +} + +// Quantile is the q-quantile (0 to 1) by nearest rank: the value the ceil(q·n)-th smallest value +// lies in the bucket of. +func (s Sketch) Quantile(q float64) float64 { + n := s.Count() + if n == 0 { + return 0 + } + rank := uint64(math.Ceil(q * float64(n))) + rank = min(max(rank, 1), n) + if rank <= s.zero { + return 0 + } + seen := s.zero + for _, b := range s.buckets { + seen += b.count + if seen >= rank { + return s.value(b.index) + } + } + return s.value(s.buckets[len(s.buckets)-1].index) +} + +// Apdex is (satisfied + tolerating/2) / n for a threshold t: satisfied values are at most t, +// tolerating ones at most 4t, a bucket counting where its value falls. +func (s Sketch) Apdex(t int64) float64 { + n := s.Count() + if n == 0 { + return 0 + } + sat, tol := float64(s.zero), 0.0 + for _, b := range s.buckets { + switch v := s.value(b.index); { + case v <= float64(t): + sat += float64(b.count) + case v <= 4*float64(t): + tol += float64(b.count) + } + } + return (sat + tol/2) / float64(n) +} + +// Encode is the stored form: the scale, the zero count, the number of buckets and then each +// bucket's index (as a delta from the one before) and count, all varints. +func (s Sketch) Encode() []byte { + out := make([]byte, 0, 3+len(s.buckets)*4) + out = append(out, byte(s.scale)) + out = binary.AppendUvarint(out, s.zero) + out = binary.AppendUvarint(out, uint64(len(s.buckets))) + prev := int32(0) + for _, b := range s.buckets { + out = binary.AppendVarint(out, int64(b.index-prev)) + out = binary.AppendUvarint(out, b.count) + prev = b.index + } + return out +} + +var errSketch = errors.New("monitor: not a sketch") + +func DecodeSketch(raw []byte) (Sketch, error) { + if len(raw) < 3 { + return Sketch{}, errSketch + } + s := Sketch{scale: int8(raw[0])} + rest := raw[1:] + read := func() (uint64, bool) { + v, n := binary.Uvarint(rest) + if n <= 0 { + return 0, false + } + rest = rest[n:] + return v, true + } + var ok bool + if s.zero, ok = read(); !ok { + return Sketch{}, errSketch + } + n, ok := read() + if !ok || n > uint64(len(rest)) { + return Sketch{}, errSketch + } + s.buckets = make([]bucket, 0, n) + prev := int64(0) + for range n { + d, k := binary.Varint(rest) + if k <= 0 { + return Sketch{}, errSketch + } + rest = rest[k:] + c, ok := read() + if !ok { + return Sketch{}, errSketch + } + prev += d + if prev < math.MinInt32 || prev > math.MaxInt32 || (len(s.buckets) > 0 && int32(prev) <= s.buckets[len(s.buckets)-1].index) { + return Sketch{}, errSketch + } + s.buckets = append(s.buckets, bucket{index: int32(prev), count: c}) + } + if len(rest) != 0 { + return Sketch{}, errSketch + } + return s, nil +} diff --git a/api/internal/monitor/sketch_test.go b/api/internal/monitor/sketch_test.go new file mode 100644 index 00000000..86820d7c --- /dev/null +++ b/api/internal/monitor/sketch_test.go @@ -0,0 +1,182 @@ +package monitor + +import ( + "math" + "math/rand/v2" + "slices" + "testing" +) + +func exactQuantile(sorted []int64, q float64) float64 { + rank := int(math.Ceil(q * float64(len(sorted)))) + rank = min(max(rank, 1), len(sorted)) + return float64(sorted[rank-1]) +} + +func durations(r *rand.Rand, n int) []int64 { + out := make([]int64, n) + for i := range out { + switch { + case i%97 == 0: + out[i] = 0 + case i%13 == 0: + out[i] = int64(math.Exp(r.NormFloat64()*0.5 + 14)) + default: + out[i] = int64(math.Exp(r.NormFloat64()*1.2+10)) + 1 + } + } + return out +} + +func TestSketchPercentilesWithinBucketWidth(t *testing.T) { + cases := []struct { + name string + scale int8 + bound float64 + seed uint64 + n int + }{ + {"durations", DurationScale, 0.011, 1, 100_000}, + {"durations, few", DurationScale, 0.011, 2, 37}, + {"sizes", SizeScale, 0.09, 3, 50_000}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := rand.New(rand.NewPCG(c.seed, 99)) + vals := durations(r, c.n) + s := NewSketch(c.scale) + for _, v := range vals { + s.Add(v, 1) + } + sorted := slices.Clone(vals) + slices.Sort(sorted) + worst := 0.0 + for _, q := range []float64{0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99, 0.999, 1} { + want, got := exactQuantile(sorted, q), s.Quantile(q) + if want == 0 { + if got != 0 { + t.Fatalf("p%v: %v, want 0", q*100, got) + } + continue + } + rel := math.Abs(got-want) / want + worst = max(worst, rel) + if rel > c.bound { + t.Fatalf("p%v: %v, exact %v, off by %.3f%%, more than %.1f%%", q*100, got, want, rel*100, c.bound*100) + } + } + t.Logf("%d values, worst error %.3f%%", c.n, worst*100) + }) + } +} + +func TestSketchMergeIsExactAndAssociative(t *testing.T) { + r := rand.New(rand.NewPCG(5, 6)) + vals := durations(r, 20_000) + whole := NewSketch(DurationScale) + parts := make([]Sketch, 1440) + for i := range parts { + parts[i] = NewSketch(DurationScale) + } + for i, v := range vals { + whole.Add(v, 1) + parts[i%len(parts)].Add(v, 1) + } + forward := NewSketch(DurationScale) + for _, p := range parts { + forward.Merge(p) + } + tree := slices.Clone(parts) + for len(tree) > 1 { + var next []Sketch + for i := 0; i < len(tree); i += 2 { + m := tree[i].Clone() + if i+1 < len(tree) { + m.Merge(tree[i+1]) + } + next = append(next, m) + } + tree = next + } + want := string(whole.Encode()) + if string(forward.Encode()) != want || string(tree[0].Encode()) != want { + t.Fatal("merged sketches differ from the sketch of every value") + } + if forward.Count() != uint64(len(vals)) { + t.Fatalf("count %d, want %d", forward.Count(), len(vals)) + } +} + +func TestSketchEncoding(t *testing.T) { + r := rand.New(rand.NewPCG(7, 8)) + s := NewSketch(DurationScale) + for _, v := range durations(r, 5000) { + s.Add(v, uint64(1+r.IntN(3))) + } + s.Add(1, 1) + s.Add(math.MaxInt64, 1) + back, err := DecodeSketch(s.Encode()) + if err != nil { + t.Fatal(err) + } + if string(back.Encode()) != string(s.Encode()) || back.Count() != s.Count() || back.Scale() != DurationScale { + t.Fatal("a decoded sketch differs from the one encoded") + } + empty, err := DecodeSketch(NewSketch(SizeScale).Encode()) + if err != nil || empty.Count() != 0 || empty.Quantile(0.5) != 0 { + t.Fatalf("empty sketch: %v %v", empty, err) + } + for _, bad := range [][]byte{nil, {5}, {5, 0}, {5, 0, 3, 1}, append(s.Encode(), 0)} { + if _, err := DecodeSketch(bad); err == nil { + t.Fatalf("%v decoded", bad) + } + } +} + +func TestSketchWeightsAndBuckets(t *testing.T) { + s := NewSketch(DurationScale) + s.Add(1000, 99) + s.Add(100_000, 1) + if s.Count() != 100 { + t.Fatalf("count %d", s.Count()) + } + if p := s.Quantile(0.99); math.Abs(p-1000)/1000 > 0.011 { + t.Fatalf("p99 %v, want about 1000", p) + } + if p := s.Quantile(1); math.Abs(p-100_000)/100_000 > 0.011 { + t.Fatalf("p100 %v, want about 100000", p) + } + b := NewSketch(DurationScale) + for _, v := range []int64{1 << 20, 1<<20 + 1, 1<<20 - 1} { + b.Add(v, 1) + if got := b.value(b.index(float64(v))); math.Abs(got-float64(v))/float64(v) > 0.011 { + t.Fatalf("%d lands in a bucket whose value is %v", v, got) + } + } +} + +func TestApdex(t *testing.T) { + const tMS = 500 + threshold := int64(tMS * 1000) + vals := []int64{100_000, 200_000, 450_000, 600_000, 1_500_000, 1_900_000, 2_500_000, 9_000_000} + s := NewSketch(DurationScale) + for _, v := range vals { + s.Add(v, 1) + } + sat, tol := 0, 0 + for _, v := range vals { + switch { + case v <= threshold: + sat++ + case v <= 4*threshold: + tol++ + } + } + want := (float64(sat) + float64(tol)/2) / float64(len(vals)) + if got := s.Apdex(threshold); got != want { + t.Fatalf("apdex %v, want %v", got, want) + } + if NewSketch(DurationScale).Apdex(threshold) != 0 { + t.Fatal("an empty sketch has no apdex") + } +} diff --git a/api/internal/oas/oas.gen.go b/api/internal/oas/oas.gen.go index 2d4d01f0..84a45dda 100644 --- a/api/internal/oas/oas.gen.go +++ b/api/internal/oas/oas.gen.go @@ -223,6 +223,7 @@ func (e AnalyticsConsentFeature) Valid() bool { // Defines values for AnalyticsDataset. const ( + AnalyticsDatasetApi AnalyticsDataset = "api" AnalyticsDatasetSupport AnalyticsDataset = "support" AnalyticsDatasetTraffic AnalyticsDataset = "traffic" ) @@ -230,6 +231,8 @@ const ( // Valid indicates whether the value is a known member of the AnalyticsDataset enum. func (e AnalyticsDataset) Valid() bool { switch e { + case AnalyticsDatasetApi: + return true case AnalyticsDatasetSupport: return true case AnalyticsDatasetTraffic: @@ -250,11 +253,14 @@ const ( AnalyticsByDay AnalyticsDimension = "time:day" AnalyticsByDevice AnalyticsDimension = "device" AnalyticsByEntryPage AnalyticsDimension = "entry_page" + AnalyticsByEnvironment AnalyticsDimension = "environment" AnalyticsByEvent AnalyticsDimension = "event" AnalyticsByExitPage AnalyticsDimension = "exit_page" AnalyticsByHostname AnalyticsDimension = "hostname" AnalyticsByHour AnalyticsDimension = "time:hour" AnalyticsByLanguage AnalyticsDimension = "language" + AnalyticsByMethod AnalyticsDimension = "method" + AnalyticsByMinute AnalyticsDimension = "time:minute" AnalyticsByMonth AnalyticsDimension = "time:month" AnalyticsByOs AnalyticsDimension = "os" AnalyticsByOsVersion AnalyticsDimension = "os_version" @@ -263,8 +269,11 @@ const ( AnalyticsByProp AnalyticsDimension = "prop" AnalyticsByReferrer AnalyticsDimension = "referrer" AnalyticsByRegion AnalyticsDimension = "region" + AnalyticsByRoute AnalyticsDimension = "route" AnalyticsByScreenSize AnalyticsDimension = "screen_size" AnalyticsBySource AnalyticsDimension = "source" + AnalyticsByStatus AnalyticsDimension = "status" + AnalyticsByStatusClass AnalyticsDimension = "status_class" AnalyticsByUtmCampaign AnalyticsDimension = "utm_campaign" AnalyticsByUtmContent AnalyticsDimension = "utm_content" AnalyticsByUtmMedium AnalyticsDimension = "utm_medium" @@ -295,6 +304,8 @@ func (e AnalyticsDimension) Valid() bool { return true case AnalyticsByEntryPage: return true + case AnalyticsByEnvironment: + return true case AnalyticsByEvent: return true case AnalyticsByExitPage: @@ -305,6 +316,10 @@ func (e AnalyticsDimension) Valid() bool { return true case AnalyticsByLanguage: return true + case AnalyticsByMethod: + return true + case AnalyticsByMinute: + return true case AnalyticsByMonth: return true case AnalyticsByOs: @@ -321,10 +336,16 @@ func (e AnalyticsDimension) Valid() bool { return true case AnalyticsByRegion: return true + case AnalyticsByRoute: + return true case AnalyticsByScreenSize: return true case AnalyticsBySource: return true + case AnalyticsByStatus: + return true + case AnalyticsByStatusClass: + return true case AnalyticsByUtmCampaign: return true case AnalyticsByUtmContent: @@ -444,10 +465,12 @@ const ( AnalyticsDimCountry AnalyticsFilterDimension = "country" AnalyticsDimDevice AnalyticsFilterDimension = "device" AnalyticsDimEntryPage AnalyticsFilterDimension = "entry_page" + AnalyticsDimEnvironment AnalyticsFilterDimension = "environment" AnalyticsDimEvent AnalyticsFilterDimension = "event" AnalyticsDimExitPage AnalyticsFilterDimension = "exit_page" AnalyticsDimHostname AnalyticsFilterDimension = "hostname" AnalyticsDimLanguage AnalyticsFilterDimension = "language" + AnalyticsDimMethod AnalyticsFilterDimension = "method" AnalyticsDimOs AnalyticsFilterDimension = "os" AnalyticsDimOsVersion AnalyticsFilterDimension = "os_version" AnalyticsDimPage AnalyticsFilterDimension = "page" @@ -455,8 +478,11 @@ const ( AnalyticsDimProp AnalyticsFilterDimension = "prop" AnalyticsDimReferrer AnalyticsFilterDimension = "referrer" AnalyticsDimRegion AnalyticsFilterDimension = "region" + AnalyticsDimRoute AnalyticsFilterDimension = "route" AnalyticsDimScreenSize AnalyticsFilterDimension = "screen_size" AnalyticsDimSource AnalyticsFilterDimension = "source" + AnalyticsDimStatus AnalyticsFilterDimension = "status" + AnalyticsDimStatusClass AnalyticsFilterDimension = "status_class" AnalyticsDimUtmCampaign AnalyticsFilterDimension = "utm_campaign" AnalyticsDimUtmContent AnalyticsFilterDimension = "utm_content" AnalyticsDimUtmMedium AnalyticsFilterDimension = "utm_medium" @@ -484,6 +510,8 @@ func (e AnalyticsFilterDimension) Valid() bool { return true case AnalyticsDimEntryPage: return true + case AnalyticsDimEnvironment: + return true case AnalyticsDimEvent: return true case AnalyticsDimExitPage: @@ -492,6 +520,8 @@ func (e AnalyticsFilterDimension) Valid() bool { return true case AnalyticsDimLanguage: return true + case AnalyticsDimMethod: + return true case AnalyticsDimOs: return true case AnalyticsDimOsVersion: @@ -506,10 +536,16 @@ func (e AnalyticsFilterDimension) Valid() bool { return true case AnalyticsDimRegion: return true + case AnalyticsDimRoute: + return true case AnalyticsDimScreenSize: return true case AnalyticsDimSource: return true + case AnalyticsDimStatus: + return true + case AnalyticsDimStatusClass: + return true case AnalyticsDimUtmCampaign: return true case AnalyticsDimUtmContent: @@ -598,33 +634,49 @@ func (e AnalyticsInsightKind) Valid() bool { // Defines values for AnalyticsMetric. const ( - AnalyticsMetricBounceRate AnalyticsMetric = "bounce_rate" - AnalyticsMetricCompletions AnalyticsMetric = "completions" - AnalyticsMetricConversations AnalyticsMetric = "conversations" - AnalyticsMetricConversionRate AnalyticsMetric = "conversion_rate" - AnalyticsMetricConversions AnalyticsMetric = "conversions" - AnalyticsMetricDiscussions AnalyticsMetric = "discussions" - AnalyticsMetricEvents AnalyticsMetric = "events" - AnalyticsMetricFeedback AnalyticsMetric = "feedback" - AnalyticsMetricHelpfulRate AnalyticsMetric = "helpful_rate" - AnalyticsMetricPageviews AnalyticsMetric = "pageviews" - AnalyticsMetricRatingsDown AnalyticsMetric = "ratings_down" - AnalyticsMetricRatingsUp AnalyticsMetric = "ratings_up" - AnalyticsMetricRevenue AnalyticsMetric = "revenue" - AnalyticsMetricScrollDepth AnalyticsMetric = "scroll_depth" - AnalyticsMetricSupportRate AnalyticsMetric = "support_rate" - AnalyticsMetricTimeOnPage AnalyticsMetric = "time_on_page" - AnalyticsMetricViewsPerVisit AnalyticsMetric = "views_per_visit" - AnalyticsMetricVisitDuration AnalyticsMetric = "visit_duration" - AnalyticsMetricVisitors AnalyticsMetric = "visitors" - AnalyticsMetricVisits AnalyticsMetric = "visits" + AnalyticsMetricAvgDuration AnalyticsMetric = "avg_duration" + AnalyticsMetricBounceRate AnalyticsMetric = "bounce_rate" + AnalyticsMetricClientErrorRate AnalyticsMetric = "client_error_rate" + AnalyticsMetricClientErrors AnalyticsMetric = "client_errors" + AnalyticsMetricCompletions AnalyticsMetric = "completions" + AnalyticsMetricConversations AnalyticsMetric = "conversations" + AnalyticsMetricConversionRate AnalyticsMetric = "conversion_rate" + AnalyticsMetricConversions AnalyticsMetric = "conversions" + AnalyticsMetricDiscussions AnalyticsMetric = "discussions" + AnalyticsMetricErrorRate AnalyticsMetric = "error_rate" + AnalyticsMetricErrors AnalyticsMetric = "errors" + AnalyticsMetricEvents AnalyticsMetric = "events" + AnalyticsMetricFeedback AnalyticsMetric = "feedback" + AnalyticsMetricHelpfulRate AnalyticsMetric = "helpful_rate" + AnalyticsMetricP50 AnalyticsMetric = "p50" + AnalyticsMetricP95 AnalyticsMetric = "p95" + AnalyticsMetricP99 AnalyticsMetric = "p99" + AnalyticsMetricPageviews AnalyticsMetric = "pageviews" + AnalyticsMetricRatingsDown AnalyticsMetric = "ratings_down" + AnalyticsMetricRatingsUp AnalyticsMetric = "ratings_up" + AnalyticsMetricRequests AnalyticsMetric = "requests" + AnalyticsMetricRequestsPerMinute AnalyticsMetric = "requests_per_minute" + AnalyticsMetricRevenue AnalyticsMetric = "revenue" + AnalyticsMetricScrollDepth AnalyticsMetric = "scroll_depth" + AnalyticsMetricSupportRate AnalyticsMetric = "support_rate" + AnalyticsMetricTimeOnPage AnalyticsMetric = "time_on_page" + AnalyticsMetricViewsPerVisit AnalyticsMetric = "views_per_visit" + AnalyticsMetricVisitDuration AnalyticsMetric = "visit_duration" + AnalyticsMetricVisitors AnalyticsMetric = "visitors" + AnalyticsMetricVisits AnalyticsMetric = "visits" ) // Valid indicates whether the value is a known member of the AnalyticsMetric enum. func (e AnalyticsMetric) Valid() bool { switch e { + case AnalyticsMetricAvgDuration: + return true case AnalyticsMetricBounceRate: return true + case AnalyticsMetricClientErrorRate: + return true + case AnalyticsMetricClientErrors: + return true case AnalyticsMetricCompletions: return true case AnalyticsMetricConversations: @@ -635,18 +687,32 @@ func (e AnalyticsMetric) Valid() bool { return true case AnalyticsMetricDiscussions: return true + case AnalyticsMetricErrorRate: + return true + case AnalyticsMetricErrors: + return true case AnalyticsMetricEvents: return true case AnalyticsMetricFeedback: return true case AnalyticsMetricHelpfulRate: return true + case AnalyticsMetricP50: + return true + case AnalyticsMetricP95: + return true + case AnalyticsMetricP99: + return true case AnalyticsMetricPageviews: return true case AnalyticsMetricRatingsDown: return true case AnalyticsMetricRatingsUp: return true + case AnalyticsMetricRequests: + return true + case AnalyticsMetricRequestsPerMinute: + return true case AnalyticsMetricRevenue: return true case AnalyticsMetricScrollDepth: @@ -707,19 +773,41 @@ func (e AnalyticsPlatform) Valid() bool { } } +// Defines values for AnalyticsPropertyKind. +const ( + AnalyticsKindApi AnalyticsPropertyKind = "api" + AnalyticsKindSite AnalyticsPropertyKind = "site" +) + +// Valid indicates whether the value is a known member of the AnalyticsPropertyKind enum. +func (e AnalyticsPropertyKind) Valid() bool { + switch e { + case AnalyticsKindApi: + return true + case AnalyticsKindSite: + return true + default: + return false + } +} + // Defines values for AnalyticsRangePeriod. const ( - AnalyticsLast12Months AnalyticsRangePeriod = "12mo" - AnalyticsLast28Days AnalyticsRangePeriod = "28d" - AnalyticsLast30Days AnalyticsRangePeriod = "30d" - AnalyticsLast6Months AnalyticsRangePeriod = "6mo" - AnalyticsLast7Days AnalyticsRangePeriod = "7d" - AnalyticsLast90Days AnalyticsRangePeriod = "90d" - AnalyticsLastMonth AnalyticsRangePeriod = "last_month" - AnalyticsThisMonth AnalyticsRangePeriod = "month" - AnalyticsThisYear AnalyticsRangePeriod = "year" - AnalyticsToday AnalyticsRangePeriod = "today" - AnalyticsYesterday AnalyticsRangePeriod = "yesterday" + AnalyticsLast12Months AnalyticsRangePeriod = "12mo" + AnalyticsLast15Minutes AnalyticsRangePeriod = "15m" + AnalyticsLast24Hours AnalyticsRangePeriod = "24h" + AnalyticsLast28Days AnalyticsRangePeriod = "28d" + AnalyticsLast30Days AnalyticsRangePeriod = "30d" + AnalyticsLast6Hours AnalyticsRangePeriod = "6h" + AnalyticsLast6Months AnalyticsRangePeriod = "6mo" + AnalyticsLast7Days AnalyticsRangePeriod = "7d" + AnalyticsLast90Days AnalyticsRangePeriod = "90d" + AnalyticsLastHour AnalyticsRangePeriod = "1h" + AnalyticsLastMonth AnalyticsRangePeriod = "last_month" + AnalyticsThisMonth AnalyticsRangePeriod = "month" + AnalyticsThisYear AnalyticsRangePeriod = "year" + AnalyticsToday AnalyticsRangePeriod = "today" + AnalyticsYesterday AnalyticsRangePeriod = "yesterday" ) // Valid indicates whether the value is a known member of the AnalyticsRangePeriod enum. @@ -727,16 +815,24 @@ func (e AnalyticsRangePeriod) Valid() bool { switch e { case AnalyticsLast12Months: return true + case AnalyticsLast15Minutes: + return true + case AnalyticsLast24Hours: + return true case AnalyticsLast28Days: return true case AnalyticsLast30Days: return true + case AnalyticsLast6Hours: + return true case AnalyticsLast6Months: return true case AnalyticsLast7Days: return true case AnalyticsLast90Days: return true + case AnalyticsLastHour: + return true case AnalyticsLastMonth: return true case AnalyticsThisMonth: @@ -797,6 +893,39 @@ func (e AnalyticsReportFrequency) Valid() bool { } } +// Defines values for AnalyticsResultMetaCapsReached. +const ( + AnalyticsCapApiVersions AnalyticsResultMetaCapsReached = "api_versions" + AnalyticsCapAppVersions AnalyticsResultMetaCapsReached = "app_versions" + AnalyticsCapConsumers AnalyticsResultMetaCapsReached = "consumers" + AnalyticsCapDimensionRows AnalyticsResultMetaCapsReached = "dimension_rows" + AnalyticsCapEnvironments AnalyticsResultMetaCapsReached = "environments" + AnalyticsCapRoutes AnalyticsResultMetaCapsReached = "routes" + AnalyticsCapServices AnalyticsResultMetaCapsReached = "services" +) + +// Valid indicates whether the value is a known member of the AnalyticsResultMetaCapsReached enum. +func (e AnalyticsResultMetaCapsReached) Valid() bool { + switch e { + case AnalyticsCapApiVersions: + return true + case AnalyticsCapAppVersions: + return true + case AnalyticsCapConsumers: + return true + case AnalyticsCapDimensionRows: + return true + case AnalyticsCapEnvironments: + return true + case AnalyticsCapRoutes: + return true + case AnalyticsCapServices: + return true + default: + return false + } +} + // Defines values for AnalyticsResultMetaServedFrom. const ( AnalyticsServedMixed AnalyticsResultMetaServedFrom = "mixed" @@ -4502,6 +4631,24 @@ func (e QueryAnalyticsParamsFormat) Valid() bool { } } +// Defines values for SendAnalyticsRequestsParamsContentEncoding. +const ( + SendAnalyticsRequestsParamsContentEncodingGzip SendAnalyticsRequestsParamsContentEncoding = "gzip" + SendAnalyticsRequestsParamsContentEncodingIdentity SendAnalyticsRequestsParamsContentEncoding = "identity" +) + +// Valid indicates whether the value is a known member of the SendAnalyticsRequestsParamsContentEncoding enum. +func (e SendAnalyticsRequestsParamsContentEncoding) Valid() bool { + switch e { + case SendAnalyticsRequestsParamsContentEncodingGzip: + return true + case SendAnalyticsRequestsParamsContentEncodingIdentity: + return true + default: + return false + } +} + // Defines values for GetAnalyticsRetentionParamsGrain. const ( AnalyticsRetentionDay GetAnalyticsRetentionParamsGrain = "day" @@ -4937,6 +5084,14 @@ type AnalyticsDataDeletion struct { // `support_rate` with pages of fewer than 100 page views after the others unless `order_by` // says otherwise. Days of the past come from the support rollup, today and days not yet rolled // up from the inbox itself. +// +// `api` is an API property's requests (`sendAnalyticsRequests`), and the only dataset an API +// property takes (`422 property_kind` otherwise, and for `api` on a site). It takes the API +// metrics of `AnalyticsMetric`, the API dimensions of `AnalyticsDimension` and filters on +// them, no goal, prop, segment, comparison or insights. Closed local days come from daily +// rows, closed hours from hourly rows and the rest from minute rows, which processes write +// every 10 seconds; `meta.served_from` is `rollups` when only daily and hourly rows answered. +// Rows are ordered by `requests` unless `order_by` says otherwise. type AnalyticsDataset string // AnalyticsDimension A filter dimension (`AnalyticsFilterDimension`), or a time grain: `time:hour` (local hours @@ -4946,6 +5101,11 @@ type AnalyticsDataset string // range is present. `event` breaks custom and built-in events down by name, `prop` by the // value of the event prop the query's `prop` names; either takes at most one other dimension, // a time grain other than `time:hour` or a visit dimension. +// +// With `dataset: api`, two of `route`, `method`, `status`, `status_class`, `environment` and +// one grain: `time:minute` (minutes, `2026-10-12T14:05` in the property's time zone, inside +// the last `YUVA_MONITOR_MINUTE_DAYS` days and at most 1,440 of them), `time:hour` (local +// hours) or `time:day`. type AnalyticsDimension string // AnalyticsDomain A host name whose pages may send events, without scheme or port: `example.com`, @@ -5187,6 +5347,12 @@ type AnalyticsExportCreate struct { // for events without it: as // a filter of visits they keep the visits with such an event; in an `event` or `prop` // breakdown they keep the matching events. + // + // With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for + // requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 + // routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response + // was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's + // environment (`production` by default, `(other)` past 20). Dimension *AnalyticsFilterDimension `json:"dimension,omitempty"` Format *AnalyticsExportFormat `json:"format,omitempty"` Kind AnalyticsExportKind `json:"kind"` @@ -5195,7 +5361,8 @@ type AnalyticsExportCreate struct { // Range Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, // `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and // `year` are the current calendar month and year to date; `last_month` the one before), or - // `from` and `to` (inclusive, at most 400 days apart). + // `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also + // be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. Range *AnalyticsRange `json:"range,omitempty"` } @@ -5231,6 +5398,12 @@ type AnalyticsFilter struct { // for events without it: as // a filter of visits they keep the visits with such an event; in an `event` or `prop` // breakdown they keep the matching events. + // + // With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for + // requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 + // routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response + // was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's + // environment (`production` by default, `(other)` past 20). Dimension AnalyticsFilterDimension `json:"dimension"` Op AnalyticsFilterOp `json:"op"` @@ -5258,6 +5431,12 @@ type AnalyticsFilterOp string // for events without it: as // a filter of visits they keep the visits with such an event; in an `event` or `prop` // breakdown they keep the matching events. +// +// With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for +// requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 +// routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response +// was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's +// environment (`production` by default, `(other)` past 20). type AnalyticsFilterDimension string // AnalyticsFunnel defines model for AnalyticsFunnel. @@ -5491,6 +5670,13 @@ type AnalyticsInsight struct { // `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), // `discussions` (discussion threads started on it) and `support_rate` (conversations per // 1,000 page views), next to `visitors` and `pageviews`. + // + // With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests + // per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no + // response was sent), `error_rate` and `client_error_rate` (percent of requests), + // `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds + // that this share of requests stayed within, from sketches accurate to 1.1%) and + // `avg_duration` (milliseconds). Metric AnalyticsMetric `json:"metric"` // Previous The comparison range's number (0 for `metric_new` and `new_value`), or the range's overall `support_rate` for a hotspot. @@ -5523,22 +5709,39 @@ type AnalyticsInsightKind string // `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), // `discussions` (discussion threads started on it) and `support_rate` (conversations per // 1,000 page views), next to `visitors` and `pageviews`. +// +// With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests +// per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no +// response was sent), `error_rate` and `client_error_rate` (percent of requests), +// `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds +// that this share of requests stayed within, from sketches accurate to 1.1%) and +// `avg_duration` (milliseconds). type AnalyticsMetric string // AnalyticsMetrics The metrics the query asked for. type AnalyticsMetrics struct { - BounceRate *float64 `json:"bounce_rate,omitempty"` - Completions *float64 `json:"completions,omitempty"` - Conversations *float64 `json:"conversations,omitempty"` - ConversionRate *float64 `json:"conversion_rate,omitempty"` - Conversions *float64 `json:"conversions,omitempty"` - Discussions *float64 `json:"discussions,omitempty"` - Events *float64 `json:"events,omitempty"` - Feedback *float64 `json:"feedback,omitempty"` - HelpfulRate *float64 `json:"helpful_rate,omitempty"` - Pageviews *float64 `json:"pageviews,omitempty"` - RatingsDown *float64 `json:"ratings_down,omitempty"` - RatingsUp *float64 `json:"ratings_up,omitempty"` + AvgDuration *float64 `json:"avg_duration,omitempty"` + BounceRate *float64 `json:"bounce_rate,omitempty"` + ClientErrorRate *float64 `json:"client_error_rate,omitempty"` + ClientErrors *float64 `json:"client_errors,omitempty"` + Completions *float64 `json:"completions,omitempty"` + Conversations *float64 `json:"conversations,omitempty"` + ConversionRate *float64 `json:"conversion_rate,omitempty"` + Conversions *float64 `json:"conversions,omitempty"` + Discussions *float64 `json:"discussions,omitempty"` + ErrorRate *float64 `json:"error_rate,omitempty"` + Errors *float64 `json:"errors,omitempty"` + Events *float64 `json:"events,omitempty"` + Feedback *float64 `json:"feedback,omitempty"` + HelpfulRate *float64 `json:"helpful_rate,omitempty"` + P50 *float64 `json:"p50,omitempty"` + P95 *float64 `json:"p95,omitempty"` + P99 *float64 `json:"p99,omitempty"` + Pageviews *float64 `json:"pageviews,omitempty"` + RatingsDown *float64 `json:"ratings_down,omitempty"` + RatingsUp *float64 `json:"ratings_up,omitempty"` + Requests *float64 `json:"requests,omitempty"` + RequestsPerMinute *float64 `json:"requests_per_minute,omitempty"` // Revenue Per currency, most first. Revenue *[]AnalyticsRevenue `json:"revenue,omitempty"` @@ -5582,6 +5785,13 @@ type AnalyticsOrder struct { // `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), // `discussions` (discussion threads started on it) and `support_rate` (conversations per // 1,000 page views), next to `visitors` and `pageviews`. + // + // With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests + // per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no + // response was sent), `error_rate` and `client_error_rate` (percent of requests), + // `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds + // that this share of requests stayed within, from sketches accurate to 1.1%) and + // `avg_duration` (milliseconds). Metric AnalyticsMetric `json:"metric"` } @@ -5608,8 +5818,11 @@ type AnalyticsPathRule struct { // apps send without an `Origin`. Server-side events (`sendAnalyticsEvents`) need no platform. type AnalyticsPlatform string -// AnalyticsProperty A website or app the workspace measures. +// AnalyticsProperty A website, app or API the workspace measures. type AnalyticsProperty struct { + // ApdexMs An API's Apdex threshold T in milliseconds; a request is satisfied within T and tolerated within 4T. + ApdexMs int32 `json:"apdex_ms"` + // Audience `children` keeps location at country level and allows no consent features or tracked props. Audience AnalyticsAudience `json:"audience"` @@ -5633,11 +5846,21 @@ type AnalyticsProperty struct { // InboxId The linked inbox, whose agents can read the property; absent when none. InboxId *uuid.UUID `json:"inbox_id,omitempty"` + // Kind `site` measures websites and apps (page views, events, visitors). `api` watches an API: its + // host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An + // API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps + // allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, + // tracked props or consent features. The kind is fixed when the property is made. + Kind AnalyticsPropertyKind `json:"kind"` + // LastEventAt When an event was last stored, to the minute. - LastEventAt *time.Time `json:"last_event_at,omitempty"` - Name string `json:"name"` - PathRules []AnalyticsPathRule `json:"path_rules"` - Platforms []AnalyticsPlatform `json:"platforms"` + LastEventAt *time.Time `json:"last_event_at,omitempty"` + + // MonitorHourDays How many days an API's hourly rows are kept; daily rows follow `retention_months`. + MonitorHourDays int32 `json:"monitor_hour_days"` + Name string `json:"name"` + PathRules []AnalyticsPathRule `json:"path_rules"` + Platforms []AnalyticsPlatform `json:"platforms"` // PreviousKeyExpiresAt After a rotation, when the previous key stops working; absent when there is none. PreviousKeyExpiresAt *time.Time `json:"previous_key_expires_at,omitempty"` @@ -5661,23 +5884,35 @@ type AnalyticsProperty struct { // AnalyticsPropertyCreate defines model for AnalyticsPropertyCreate. type AnalyticsPropertyCreate struct { + ApdexMs *int32 `json:"apdex_ms,omitempty"` + // Audience `children` keeps location at country level and allows no consent features or tracked props. Audience *AnalyticsAudience `json:"audience,omitempty"` // Collection `anonymous` counts without consent: nothing is put on the visitor's device and no IP // address is stored. `consent` takes an event only when it carries `c.a: 1`, which the // script sends after `yuva.consent({analytics: true})`. - Collection *AnalyticsCollection `json:"collection,omitempty"` - ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` - Currency *string `json:"currency,omitempty"` - Domains []AnalyticsDomain `json:"domains"` - ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` - ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` - InboxId *uuid.UUID `json:"inbox_id,omitempty"` - Name string `json:"name"` - PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` - - // Platforms `[web]` when absent. + Collection *AnalyticsCollection `json:"collection,omitempty"` + ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` + Currency *string `json:"currency,omitempty"` + + // Domains 1 to 50 host names for a site, 0 to 50 for an API. + Domains []AnalyticsDomain `json:"domains"` + ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` + ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` + InboxId *uuid.UUID `json:"inbox_id,omitempty"` + + // Kind `site` measures websites and apps (page views, events, visitors). `api` watches an API: its + // host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An + // API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps + // allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, + // tracked props or consent features. The kind is fixed when the property is made. + Kind *AnalyticsPropertyKind `json:"kind,omitempty"` + MonitorHourDays *int32 `json:"monitor_hour_days,omitempty"` + Name string `json:"name"` + PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` + + // Platforms At least one for a site, `[web]` when absent; for an API `ios` and `android` only, none when absent. Platforms *[]AnalyticsPlatform `json:"platforms,omitempty"` // RetentionMonths 25 when absent; `null` keeps rollups forever. @@ -5694,6 +5929,13 @@ type AnalyticsPropertyDeletion struct { Name string `json:"name"` } +// AnalyticsPropertyKind `site` measures websites and apps (page views, events, visitors). `api` watches an API: its +// host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An +// API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps +// allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, +// tracked props or consent features. The kind is fixed when the property is made. +type AnalyticsPropertyKind string + // AnalyticsPropertyList defines model for AnalyticsPropertyList. type AnalyticsPropertyList struct { Items []AnalyticsProperty `json:"items"` @@ -5710,24 +5952,31 @@ type AnalyticsPropertyRef struct { // AnalyticsPropertyUpdate defines model for AnalyticsPropertyUpdate. type AnalyticsPropertyUpdate struct { + ApdexMs *int32 `json:"apdex_ms,omitempty"` + // Audience `children` keeps location at country level and allows no consent features or tracked props. Audience *AnalyticsAudience `json:"audience,omitempty"` // Collection `anonymous` counts without consent: nothing is put on the visitor's device and no IP // address is stored. `consent` takes an event only when it carries `c.a: 1`, which the // script sends after `yuva.consent({analytics: true})`. - Collection *AnalyticsCollection `json:"collection,omitempty"` - ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` - Currency *string `json:"currency,omitempty"` - Domains *[]AnalyticsDomain `json:"domains,omitempty"` - ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` - ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` + Collection *AnalyticsCollection `json:"collection,omitempty"` + ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` + Currency *string `json:"currency,omitempty"` + + // Domains 1 to 50 host names for a site, 0 to 50 for an API. + Domains *[]AnalyticsDomain `json:"domains,omitempty"` + ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` + ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` // InboxId `null` unlinks the property from its inbox. - InboxId nullable.Nullable[uuid.UUID] `json:"inbox_id,omitempty"` - Name *string `json:"name,omitempty"` - PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` - Platforms *[]AnalyticsPlatform `json:"platforms,omitempty"` + InboxId nullable.Nullable[uuid.UUID] `json:"inbox_id,omitempty"` + MonitorHourDays *int32 `json:"monitor_hour_days,omitempty"` + Name *string `json:"name,omitempty"` + PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` + + // Platforms At least one for a site; for an API `ios` and `android` only. + Platforms *[]AnalyticsPlatform `json:"platforms,omitempty"` // RetentionMonths `null` keeps rollups forever. RetentionMonths nullable.Nullable[int32] `json:"retention_months,omitempty"` @@ -5755,6 +6004,14 @@ type AnalyticsQuery struct { // `support_rate` with pages of fewer than 100 page views after the others unless `order_by` // says otherwise. Days of the past come from the support rollup, today and days not yet rolled // up from the inbox itself. + // + // `api` is an API property's requests (`sendAnalyticsRequests`), and the only dataset an API + // property takes (`422 property_kind` otherwise, and for `api` on a site). It takes the API + // metrics of `AnalyticsMetric`, the API dimensions of `AnalyticsDimension` and filters on + // them, no goal, prop, segment, comparison or insights. Closed local days come from daily + // rows, closed hours from hourly rows and the rest from minute rows, which processes write + // every 10 seconds; `meta.served_from` is `rollups` when only daily and hourly rows answered. + // Rows are ordered by `requests` unless `order_by` says otherwise. Dataset *AnalyticsDataset `json:"dataset,omitempty"` Dimensions *[]AnalyticsDimension `json:"dimensions,omitempty"` Filters *[]AnalyticsFilter `json:"filters,omitempty"` @@ -5782,7 +6039,8 @@ type AnalyticsQuery struct { // Range Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, // `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and // `year` are the current calendar month and year to date; `last_month` the one before), or - // `from` and `to` (inclusive, at most 400 days apart). + // `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also + // be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. Range AnalyticsRange `json:"range"` // SegmentId A saved segment of the property, whose filters are added to `filters`. @@ -5792,7 +6050,8 @@ type AnalyticsQuery struct { // AnalyticsRange Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, // `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and // `year` are the current calendar month and year to date; `last_month` the one before), or -// `from` and `to` (inclusive, at most 400 days apart). +// `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also +// be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. type AnalyticsRange struct { From *openapi_types.Date `json:"from,omitempty"` Period *AnalyticsRangePeriod `json:"period,omitempty"` @@ -5894,6 +6153,74 @@ type AnalyticsRealtime struct { // AnalyticsReportFrequency defines model for AnalyticsReportFrequency. type AnalyticsReportFrequency string +// AnalyticsRequestRecord One request an API served. Give `route`, or `path` when the router knows no pattern. +type AnalyticsRequestRecord struct { + ApiVersion *string `json:"api_version,omitempty"` + + // Consumer Who called, as you name them, such as an API key's name or a client id; never a person's address (e-mail-shaped values become `(scrubbed)`). + Consumer *string `json:"consumer,omitempty"` + + // DurationUs How long the request took, in microseconds. + DurationUs int64 `json:"duration_us"` + + // Environment `production` when absent. + Environment *string `json:"environment,omitempty"` + + // Ip The caller's address, only to find its country; shortened on arrival, never stored. + Ip *string `json:"ip,omitempty"` + + // Method The request method; anything but the RFC 9110 methods and `PATCH` counts as `_OTHER`. + // + // Examples: GET + Method string `json:"method"` + + // Path The request path, when there is no `route`; only the pattern inferred from it is kept. + // + // Examples: /users/42 + Path *string `json:"path,omitempty"` + + // Release The service's release, for deploy markers. + Release *string `json:"release,omitempty"` + + // RequestSize Bytes of the request body. + RequestSize *int64 `json:"request_size,omitempty"` + + // ResponseSize Bytes of the response body. + ResponseSize *int64 `json:"response_size,omitempty"` + + // Route The route pattern that matched, starting with `/`, without method or host. + // + // Examples: /users/{id} + Route *string `json:"route,omitempty"` + + // SampleRate This record stands for this many requests, for a host that sends one in N successful + // requests. Only records with a status of 1 to 399 may be sampled. + SampleRate *int `json:"sample_rate,omitempty"` + + // Service The service that served the request. + Service *string `json:"service,omitempty"` + + // Status The response status, 100 to 599, or 0 when no response was sent. + Status int `json:"status"` + + // Time When the request started; now when absent. Records older than 72 hours are not counted. + Time *time.Time `json:"time,omitempty"` +} + +// AnalyticsRequestRecords defines model for AnalyticsRequestRecords. +type AnalyticsRequestRecords struct { + Records []AnalyticsRequestRecord `json:"records"` +} + +// AnalyticsRequestsResult defines model for AnalyticsRequestsResult. +type AnalyticsRequestsResult struct { + // Accepted The valid records, counted unless a rule of the operation dropped them. + Accepted int `json:"accepted"` + + // Errors The records left out as invalid, by their position in `records`. + Errors []AnalyticsEventError `json:"errors"` +} + // AnalyticsResult defines model for AnalyticsResult. type AnalyticsResult struct { Comparison *AnalyticsComparison `json:"comparison,omitempty"` @@ -5914,7 +6241,15 @@ type AnalyticsResult struct { // AnalyticsResultMeta defines model for AnalyticsResultMeta. type AnalyticsResultMeta struct { - From openapi_types.Date `json:"from"` + // CapsReached `dataset: api`: the caps the property has reached, so new values count as `(other)`: + // `routes` (1,000), `consumers` (10,000), `api_versions` (50), `app_versions` (200), + // `environments` (20), `services` (100 per workspace), `dimension_rows` (an hour of the + // range reached 50,000 consumer, country or app version rows). + CapsReached *[]AnalyticsResultMetaCapsReached `json:"caps_reached,omitempty"` + + // Estimated `dataset: api`: some requests of the range were sampled (`sample_rate`), so counts are estimates. + Estimated *bool `json:"estimated,omitempty"` + From openapi_types.Date `json:"from"` // RawWindowFrom The first day raw rows are kept for, which filters and breakdowns the rollups cannot answer need. RawWindowFrom openapi_types.Date `json:"raw_window_from"` @@ -5923,6 +6258,9 @@ type AnalyticsResultMeta struct { To openapi_types.Date `json:"to"` } +// AnalyticsResultMetaCapsReached defines model for AnalyticsResultMeta.CapsReached. +type AnalyticsResultMetaCapsReached string + // AnalyticsResultMetaServedFrom defines model for AnalyticsResultMeta.ServedFrom. type AnalyticsResultMetaServedFrom string @@ -6250,6 +6588,12 @@ type AnalyticsUsage struct { // Month Examples: 2026-10 Month string `json:"month"` + + // Requests API request records counted this month (`sendAnalyticsRequests`); absent when API monitoring is off. + Requests *int64 `json:"requests,omitempty"` + + // RequestsLimit The monthly cap on request records; absent when there is none. Records over it are not counted until the month ends. + RequestsLimit *int64 `json:"requests_limit,omitempty"` } // AnalyticsVisitPage defines model for AnalyticsVisitPage. @@ -6351,8 +6695,8 @@ type ApiKeyList struct { // Yuva. Owners and admins only; never offered to agents' tokens, refused for keys limited // to inboxes. // - `analytics:read`: analytics properties. -// - `analytics:write`: server-side analytics events; for API keys only, never offered to -// OAuth clients. +// - `analytics:write`: server-side analytics events and API request records; for API keys +// only, never offered to OAuth clients. // - `analytics:manage`: create, change and delete analytics properties, rotate their keys, // delete their data. Never offered to agents' tokens, refused for keys limited to inboxes. // @@ -14166,6 +14510,15 @@ type GetAnalyticsRealtimeParams struct { YuvaWorkspace *WorkspaceHeader `json:"Yuva-Workspace,omitempty"` } +// SendAnalyticsRequestsParams defines parameters for SendAnalyticsRequests. +type SendAnalyticsRequestsParams struct { + // ContentEncoding `gzip` when the body is compressed. + ContentEncoding *SendAnalyticsRequestsParamsContentEncoding `json:"Content-Encoding,omitempty"` +} + +// SendAnalyticsRequestsParamsContentEncoding defines parameters for SendAnalyticsRequests. +type SendAnalyticsRequestsParamsContentEncoding string + // GetAnalyticsRetentionParams defines parameters for GetAnalyticsRetention. type GetAnalyticsRetentionParams struct { // Grain `week` when absent. @@ -16308,6 +16661,9 @@ type UpdateAnalyticsGoalJSONRequestBody = AnalyticsGoalUpdate // QueryAnalyticsJSONRequestBody defines body for QueryAnalytics for application/json ContentType. type QueryAnalyticsJSONRequestBody = AnalyticsQuery +// SendAnalyticsRequestsJSONRequestBody defines body for SendAnalyticsRequests for application/json ContentType. +type SendAnalyticsRequestsJSONRequestBody = AnalyticsRequestRecords + // CreateAnalyticsSegmentJSONRequestBody defines body for CreateAnalyticsSegment for application/json ContentType. type CreateAnalyticsSegmentJSONRequestBody = AnalyticsSegmentCreate @@ -19932,6 +20288,9 @@ type ServerInterface interface { // GetAnalyticsRealtime The last 30 minutes // (GET /v1/analytics/properties/{propertyId}/realtime) GetAnalyticsRealtime(w http.ResponseWriter, r *http.Request, propertyId AnalyticsPropertyId, params GetAnalyticsRealtimeParams) + // SendAnalyticsRequests Send an API's request records + // (POST /v1/analytics/properties/{propertyId}/requests) + SendAnalyticsRequests(w http.ResponseWriter, r *http.Request, propertyId AnalyticsPropertyId, params SendAnalyticsRequestsParams) // GetAnalyticsRetention How consented visitors come back // (GET /v1/analytics/properties/{propertyId}/retention) GetAnalyticsRetention(w http.ResponseWriter, r *http.Request, propertyId AnalyticsPropertyId, params GetAnalyticsRetentionParams) @@ -23836,6 +24195,56 @@ func (siw *ServerInterfaceWrapper) GetAnalyticsRealtime(w http.ResponseWriter, r handler.ServeHTTP(w, r) } +// SendAnalyticsRequests operation middleware +func (siw *ServerInterfaceWrapper) SendAnalyticsRequests(w http.ResponseWriter, r *http.Request) { + + var err error + _ = err + + // ------------- Path parameter "propertyId" ------------- + var propertyId AnalyticsPropertyId + + err = runtime.BindStyledParameterWithOptions("simple", "propertyId", r.PathValue("propertyId"), &propertyId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "uuid", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "propertyId", Err: err}) + return + } + + // Parameter object where we will unmarshal all parameters from the context + var params SendAnalyticsRequestsParams + + headers := r.Header + + // ------------- Optional header parameter "Content-Encoding" ------------- + if valueList, found := headers[http.CanonicalHeaderKey("Content-Encoding")]; found { + var ContentEncoding SendAnalyticsRequestsParamsContentEncoding + n := len(valueList) + if n != 1 { + siw.ErrorHandlerFunc(w, r, &TooManyValuesForParamError{ParamName: "Content-Encoding", Count: n}) + return + } + + err = runtime.BindStyledParameterWithOptions("simple", "Content-Encoding", valueList[0], &ContentEncoding, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationHeader, Explode: false, Required: false, Type: "string", Format: ""}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "Content-Encoding", Err: err}) + return + } + + params.ContentEncoding = &ContentEncoding + + } + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.SendAnalyticsRequests(w, r, propertyId, params) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // GetAnalyticsRetention operation middleware func (siw *ServerInterfaceWrapper) GetAnalyticsRetention(w http.ResponseWriter, r *http.Request) { @@ -37496,6 +37905,7 @@ func HandlerWithOptions(si ServerInterface, options StdHTTPServerOptions) http.H m.HandleFunc(http.MethodPost+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/query", wrapper.QueryAnalytics) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/realtime", wrapper.GetAnalyticsRealtime) m.HandleFunc(http.MethodPost+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/events", wrapper.SendAnalyticsEvents) + m.HandleFunc(http.MethodPost+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/requests", wrapper.SendAnalyticsRequests) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/retention", wrapper.GetAnalyticsRetention) m.HandleFunc(http.MethodDelete+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/visitors/{persistentId}", wrapper.DeleteAnalyticsVisitor) m.HandleFunc(http.MethodGet+" "+options.BaseURL+"/v1/analytics/properties/{propertyId}/health", wrapper.GetAnalyticsHealth) @@ -45263,6 +45673,116 @@ func (response GetAnalyticsRealtime429ApplicationProblemPlusJSONResponse) VisitG return err } +type SendAnalyticsRequestsRequestObject struct { + PropertyId AnalyticsPropertyId `json:"propertyId"` + Params SendAnalyticsRequestsParams + Body *SendAnalyticsRequestsJSONRequestBody +} + +type SendAnalyticsRequestsResponseObject interface { + VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error +} + +type SendAnalyticsRequests202JSONResponse AnalyticsRequestsResult + +func (response SendAnalyticsRequests202JSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(202) + _, err := buf.WriteTo(w) + return err +} + +type SendAnalyticsRequests400ApplicationProblemPlusJSONResponse struct { + ProblemApplicationProblemPlusJSONResponse +} + +func (response SendAnalyticsRequests400ApplicationProblemPlusJSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(400) + _, err := buf.WriteTo(w) + return err +} + +type SendAnalyticsRequests401ApplicationProblemPlusJSONResponse Problem + +func (response SendAnalyticsRequests401ApplicationProblemPlusJSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(401) + _, err := buf.WriteTo(w) + return err +} + +type SendAnalyticsRequests403ApplicationProblemPlusJSONResponse Problem + +func (response SendAnalyticsRequests403ApplicationProblemPlusJSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(403) + _, err := buf.WriteTo(w) + return err +} + +type SendAnalyticsRequests404ApplicationProblemPlusJSONResponse Problem + +func (response SendAnalyticsRequests404ApplicationProblemPlusJSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(404) + _, err := buf.WriteTo(w) + return err +} + +type SendAnalyticsRequests422ApplicationProblemPlusJSONResponse Problem + +func (response SendAnalyticsRequests422ApplicationProblemPlusJSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(422) + _, err := buf.WriteTo(w) + return err +} + +type SendAnalyticsRequests503ApplicationProblemPlusJSONResponse Problem + +func (response SendAnalyticsRequests503ApplicationProblemPlusJSONResponse) VisitSendAnalyticsRequestsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(503) + _, err := buf.WriteTo(w) + return err +} + type GetAnalyticsRetentionRequestObject struct { PropertyId AnalyticsPropertyId `json:"propertyId"` Params GetAnalyticsRetentionParams @@ -63293,6 +63813,9 @@ type StrictServerInterface interface { // GetAnalyticsRealtime The last 30 minutes // (GET /v1/analytics/properties/{propertyId}/realtime) GetAnalyticsRealtime(ctx context.Context, request GetAnalyticsRealtimeRequestObject) (GetAnalyticsRealtimeResponseObject, error) + // SendAnalyticsRequests Send an API's request records + // (POST /v1/analytics/properties/{propertyId}/requests) + SendAnalyticsRequests(ctx context.Context, request SendAnalyticsRequestsRequestObject) (SendAnalyticsRequestsResponseObject, error) // GetAnalyticsRetention How consented visitors come back // (GET /v1/analytics/properties/{propertyId}/retention) GetAnalyticsRetention(ctx context.Context, request GetAnalyticsRetentionRequestObject) (GetAnalyticsRetentionResponseObject, error) @@ -66322,6 +66845,40 @@ func (sh *strictHandler) GetAnalyticsRealtime(w http.ResponseWriter, r *http.Req } } +// SendAnalyticsRequests operation middleware +func (sh *strictHandler) SendAnalyticsRequests(w http.ResponseWriter, r *http.Request, propertyId AnalyticsPropertyId, params SendAnalyticsRequestsParams) { + var request SendAnalyticsRequestsRequestObject + + request.PropertyId = propertyId + request.Params = params + + var body SendAnalyticsRequestsJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.SendAnalyticsRequests(ctx, request.(SendAnalyticsRequestsRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "SendAnalyticsRequests") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(SendAnalyticsRequestsResponseObject); ok { + if err := validResponse.VisitSendAnalyticsRequestsResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // GetAnalyticsRetention operation middleware func (sh *strictHandler) GetAnalyticsRetention(w http.ResponseWriter, r *http.Request, propertyId AnalyticsPropertyId, params GetAnalyticsRetentionParams) { var request GetAnalyticsRetentionRequestObject diff --git a/api/internal/store/analytics.sql.gen.go b/api/internal/store/analytics.sql.gen.go index e1c21d6a..09c825db 100644 --- a/api/internal/store/analytics.sql.gen.go +++ b/api/internal/store/analytics.sql.gen.go @@ -295,19 +295,20 @@ func (q *Queries) CreateAnalyticsGoal(ctx context.Context, arg CreateAnalyticsGo } const createAnalyticsProperty = `-- name: CreateAnalyticsProperty :one -INSERT INTO analytics_properties (id, workspace_id, inbox_id, name, public_key, domains, platforms, timezone, currency, collection, - consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, created_by, - created_at, updated_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11, $12, $13, $14, $15, $16, $17, $18, - $19, $20, $20) -RETURNING id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at +INSERT INTO analytics_properties (id, workspace_id, inbox_id, kind, name, public_key, domains, platforms, timezone, currency, collection, + consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, apdex_ms, monitor_hour_days, + visitor_key, created_by, created_at, updated_at) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, + $12, $13, $14, $15, $16, $17, $18, $19, + $20, $21, $22, $23, $23) +RETURNING id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days ` type CreateAnalyticsPropertyParams struct { ID uuid.UUID WorkspaceID uuid.UUID InboxID *uuid.UUID + Kind string Name string PublicKey string Domains []string @@ -322,6 +323,8 @@ type CreateAnalyticsPropertyParams struct { PathRules []byte TrackedProps []string RetentionMonths *int32 + ApdexMs int32 + MonitorHourDays int32 VisitorKey []byte CreatedBy *uuid.UUID Now time.Time @@ -332,6 +335,7 @@ func (q *Queries) CreateAnalyticsProperty(ctx context.Context, arg CreateAnalyti arg.ID, arg.WorkspaceID, arg.InboxID, + arg.Kind, arg.Name, arg.PublicKey, arg.Domains, @@ -346,6 +350,8 @@ func (q *Queries) CreateAnalyticsProperty(ctx context.Context, arg CreateAnalyti arg.PathRules, arg.TrackedProps, arg.RetentionMonths, + arg.ApdexMs, + arg.MonitorHourDays, arg.VisitorKey, arg.CreatedBy, arg.Now, @@ -381,6 +387,9 @@ func (q *Queries) CreateAnalyticsProperty(ctx context.Context, arg CreateAnalyti &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } @@ -587,7 +596,7 @@ func (q *Queries) DeleteStaleAnalyticsVisitClaims(ctx context.Context, arg Delet } const findAnalyticsPropertyByKey = `-- name: FindAnalyticsPropertyByKey :one -SELECT p.id, p.workspace_id, p.inbox_id, p.name, p.public_key, p.previous_key, p.previous_key_until, p.domains, p.platforms, p.timezone, p.currency, p.collection, p.consent_features, p.audience, p.excluded_paths, p.excluded_ips, p.path_rules, p.tracked_props, p.retention_months, p.visitor_key, p.rollup_version, p.first_event_at, p.last_event_at, p.created_by, p.created_at, p.updated_at, p.deleted_at, p.goals_version, p.cohorts_built_at FROM analytics_properties p JOIN workspaces w ON w.id = p.workspace_id +SELECT p.id, p.workspace_id, p.inbox_id, p.name, p.public_key, p.previous_key, p.previous_key_until, p.domains, p.platforms, p.timezone, p.currency, p.collection, p.consent_features, p.audience, p.excluded_paths, p.excluded_ips, p.path_rules, p.tracked_props, p.retention_months, p.visitor_key, p.rollup_version, p.first_event_at, p.last_event_at, p.created_by, p.created_at, p.updated_at, p.deleted_at, p.goals_version, p.cohorts_built_at, p.kind, p.apdex_ms, p.monitor_hour_days FROM analytics_properties p JOIN workspaces w ON w.id = p.workspace_id WHERE (p.public_key = $1 OR (p.previous_key = $1 AND p.previous_key_until > $2)) AND p.deleted_at IS NULL AND w.deleted_at IS NULL ` @@ -632,6 +641,9 @@ func (q *Queries) FindAnalyticsPropertyByKey(ctx context.Context, arg FindAnalyt &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } @@ -726,7 +738,7 @@ func (q *Queries) GetAnalyticsGoal(ctx context.Context, arg GetAnalyticsGoalPara } const getAnalyticsProperty = `-- name: GetAnalyticsProperty :one -SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at FROM analytics_properties WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL +SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days FROM analytics_properties WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL ` type GetAnalyticsPropertyParams struct { @@ -767,6 +779,9 @@ func (q *Queries) GetAnalyticsProperty(ctx context.Context, arg GetAnalyticsProp &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } @@ -824,7 +839,7 @@ func (q *Queries) GetAnalyticsUsage(ctx context.Context, arg GetAnalyticsUsagePa } const getDeletedAnalyticsProperty = `-- name: GetDeletedAnalyticsProperty :one -SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at FROM analytics_properties WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NOT NULL +SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days FROM analytics_properties WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NOT NULL ` type GetDeletedAnalyticsPropertyParams struct { @@ -865,6 +880,9 @@ func (q *Queries) GetDeletedAnalyticsProperty(ctx context.Context, arg GetDelete &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } @@ -1131,7 +1149,7 @@ func (q *Queries) ListAnalyticsIngestStats(ctx context.Context, arg ListAnalytic } const listAnalyticsProperties = `-- name: ListAnalyticsProperties :many -SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at FROM analytics_properties p +SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days FROM analytics_properties p WHERE p.workspace_id = $1 AND p.deleted_at IS NULL AND ($2::bool OR EXISTS ( SELECT 1 FROM inbox_viewers iv @@ -1184,6 +1202,9 @@ func (q *Queries) ListAnalyticsProperties(ctx context.Context, arg ListAnalytics &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ); err != nil { return nil, err } @@ -1462,7 +1483,7 @@ func (q *Queries) ListWorkspaceAnalyticsPropertyIDs(ctx context.Context, workspa } const lockAnalyticsProperty = `-- name: LockAnalyticsProperty :one -SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at FROM analytics_properties WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE +SELECT id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days FROM analytics_properties WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE ` type LockAnalyticsPropertyParams struct { @@ -1503,6 +1524,9 @@ func (q *Queries) LockAnalyticsProperty(ctx context.Context, arg LockAnalyticsPr &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } @@ -1574,7 +1598,7 @@ const rotateAnalyticsKey = `-- name: RotateAnalyticsKey :one UPDATE analytics_properties SET previous_key = public_key, previous_key_until = $1, public_key = $2, updated_at = $3 WHERE workspace_id = $4 AND id = $5 AND deleted_at IS NULL -RETURNING id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at +RETURNING id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days ` type RotateAnalyticsKeyParams struct { @@ -1624,6 +1648,9 @@ func (q *Queries) RotateAnalyticsKey(ctx context.Context, arg RotateAnalyticsKey &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } @@ -1891,9 +1918,9 @@ UPDATE analytics_properties SET inbox_id = $1, name = $2, domains = $3, platforms = $4, timezone = $5, currency = $6, collection = $7, consent_features = $8, audience = $9, excluded_paths = $10, excluded_ips = $11, path_rules = $12, tracked_props = $13, - retention_months = $14, updated_at = $15 -WHERE workspace_id = $16 AND id = $17 AND deleted_at IS NULL -RETURNING id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at + retention_months = $14, apdex_ms = $15, monitor_hour_days = $16, updated_at = $17 +WHERE workspace_id = $18 AND id = $19 AND deleted_at IS NULL +RETURNING id, workspace_id, inbox_id, name, public_key, previous_key, previous_key_until, domains, platforms, timezone, currency, collection, consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, rollup_version, first_event_at, last_event_at, created_by, created_at, updated_at, deleted_at, goals_version, cohorts_built_at, kind, apdex_ms, monitor_hour_days ` type UpdateAnalyticsPropertyParams struct { @@ -1911,6 +1938,8 @@ type UpdateAnalyticsPropertyParams struct { PathRules []byte TrackedProps []string RetentionMonths *int32 + ApdexMs int32 + MonitorHourDays int32 Now time.Time WorkspaceID uuid.UUID ID uuid.UUID @@ -1932,6 +1961,8 @@ func (q *Queries) UpdateAnalyticsProperty(ctx context.Context, arg UpdateAnalyti arg.PathRules, arg.TrackedProps, arg.RetentionMonths, + arg.ApdexMs, + arg.MonitorHourDays, arg.Now, arg.WorkspaceID, arg.ID, @@ -1967,6 +1998,9 @@ func (q *Queries) UpdateAnalyticsProperty(ctx context.Context, arg UpdateAnalyti &i.DeletedAt, &i.GoalsVersion, &i.CohortsBuiltAt, + &i.Kind, + &i.ApdexMs, + &i.MonitorHourDays, ) return i, err } diff --git a/api/internal/store/broadcasts.sql.gen.go b/api/internal/store/broadcasts.sql.gen.go index 73eeb366..e3c80bc8 100644 --- a/api/internal/store/broadcasts.sql.gen.go +++ b/api/internal/store/broadcasts.sql.gen.go @@ -2104,7 +2104,7 @@ func (q *Queries) SetConversationBroadcast(ctx context.Context, arg SetConversat } const setWorkspaceBroadcastDailyLimit = `-- name: SetWorkspaceBroadcastDailyLimit :one -UPDATE workspaces SET broadcast_daily_limit = $1 WHERE id = $2 RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events +UPDATE workspaces SET broadcast_daily_limit = $1 WHERE id = $2 RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests ` type SetWorkspaceBroadcastDailyLimitParams struct { @@ -2130,12 +2130,13 @@ func (q *Queries) SetWorkspaceBroadcastDailyLimit(ctx context.Context, arg SetWo &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } const setWorkspaceBroadcastsByKeys = `-- name: SetWorkspaceBroadcastsByKeys :one -UPDATE workspaces SET broadcasts_by_keys = $1 WHERE id = $2 RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events +UPDATE workspaces SET broadcasts_by_keys = $1 WHERE id = $2 RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests ` type SetWorkspaceBroadcastsByKeysParams struct { @@ -2161,6 +2162,7 @@ func (q *Queries) SetWorkspaceBroadcastsByKeys(ctx context.Context, arg SetWorks &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } @@ -2168,7 +2170,7 @@ func (q *Queries) SetWorkspaceBroadcastsByKeys(ctx context.Context, arg SetWorks const setWorkspaceBroadcastsStatus = `-- name: SetWorkspaceBroadcastsStatus :one UPDATE workspaces SET broadcasts_status = $1, broadcasts_status_reason = $2, broadcasts_status_at = $3::timestamptz WHERE id = $4 -RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events +RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests ` type SetWorkspaceBroadcastsStatusParams struct { @@ -2201,6 +2203,7 @@ func (q *Queries) SetWorkspaceBroadcastsStatus(ctx context.Context, arg SetWorks &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } diff --git a/api/internal/store/migrations/00056_monitor.sql b/api/internal/store/migrations/00056_monitor.sql new file mode 100644 index 00000000..04b188b5 --- /dev/null +++ b/api/internal/store/migrations/00056_monitor.sql @@ -0,0 +1,245 @@ +-- +goose Up +-- API monitoring (M16.1). An analytics property of kind api watches an API: its host sends a record +-- per request, which each process adds up in memory and writes as rows per minute and hour, never a +-- row per request and never an address. Its domains (0 to 50) are the API's host names and its +-- platforms the apps that may send app-side records; it has no goals, props or consent features. +ALTER TABLE analytics_properties + ADD COLUMN kind text NOT NULL DEFAULT 'site' CHECK (kind IN ('site', 'api')), + ADD COLUMN apdex_ms integer NOT NULL DEFAULT 500 CHECK (apdex_ms BETWEEN 1 AND 60000), + ADD COLUMN monitor_hour_days integer NOT NULL DEFAULT 90 CHECK (monitor_hour_days BETWEEN 7 AND 400), + DROP CONSTRAINT analytics_properties_domains_check, + ADD CONSTRAINT analytics_properties_domains_check CHECK (cardinality(domains) <= 50 AND (kind = 'api' OR cardinality(domains) >= 1)), + DROP CONSTRAINT analytics_properties_platforms_check, + ADD CONSTRAINT analytics_properties_platforms_check CHECK (platforms <@ ARRAY['web', 'ios', 'android'] + AND CASE WHEN kind = 'api' THEN NOT ('web' = ANY (platforms)) ELSE cardinality(platforms) >= 1 END), + ADD CONSTRAINT analytics_properties_api_check CHECK (kind = 'site' OR (consent_features = '{}' AND tracked_props = '{}')); + +-- Dictionaries give the rows integer ids. A route is a pattern such as /users/{id}; inferred ones +-- came from a path. (other) and (unmatched) are rows too but do not count towards the caps. +CREATE TABLE monitor_routes ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + id integer NOT NULL, + pattern text COLLATE "C" NOT NULL, + inferred boolean NOT NULL DEFAULT false, + hidden boolean NOT NULL DEFAULT false, + apdex_ms integer CHECK (apdex_ms BETWEEN 1 AND 60000), + first_seen_at timestamptz NOT NULL, + last_seen_at timestamptz NOT NULL, + PRIMARY KEY (property_id, id), + UNIQUE (property_id, pattern), + FOREIGN KEY (workspace_id, property_id) REFERENCES analytics_properties (workspace_id, id) ON DELETE CASCADE +); + +-- Consumers, API versions, app versions, environments, services and countries of a property. Id 0 +-- is the empty value and has no row. +CREATE TABLE monitor_names ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + kind text NOT NULL CHECK (kind IN ('environment', 'service', 'api_version', 'consumer', 'app_version', 'country')), + id integer NOT NULL CHECK (id > 0), + value text COLLATE "C" NOT NULL, + first_seen_at timestamptz NOT NULL, + PRIMARY KEY (property_id, kind, id), + UNIQUE (property_id, kind, value), + FOREIGN KEY (workspace_id, property_id) REFERENCES analytics_properties (workspace_id, id) ON DELETE CASCADE +); +CREATE INDEX monitor_names_services_idx ON monitor_names (workspace_id, kind, value); + +-- Core rows by UTC minute, one per process (writer) and key. A process upserts its cumulative totals +-- every 10 seconds, replacing its previous row, so processes never share a row. durations and +-- response_sizes are sketches (log-bucketed histograms) that only Go opens. Day partitions, made +-- and dropped by the aggregator and the analytics_partitions job; no foreign keys, a property's rows +-- are deleted explicitly. +CREATE TABLE monitor_minute ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + minute timestamptz NOT NULL, + writer bigint NOT NULL, + environment integer NOT NULL, + service integer NOT NULL, + route integer NOT NULL, + method smallint NOT NULL, + status smallint NOT NULL, + api_version integer NOT NULL, + source smallint NOT NULL, + platform smallint NOT NULL, + requests bigint NOT NULL, + errors bigint NOT NULL, + client_errors bigint NOT NULL, + sampled bigint NOT NULL, + duration_sum bigint NOT NULL, + duration_min bigint NOT NULL, + duration_max bigint NOT NULL, + durations bytea NOT NULL, + request_bytes bigint NOT NULL, + response_bytes bigint NOT NULL, + response_sizes bytea NOT NULL, + PRIMARY KEY (property_id, minute, writer, environment, service, route, method, status, api_version, source, platform) +) PARTITION BY RANGE (minute); + +-- Core rows by the property's local hour (hour is the hour's first instant), folded from the minute +-- rows once the hour is closed. Month partitions; rows past the property's monitor_hour_days are +-- deleted. +CREATE TABLE monitor_hour ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + hour timestamptz NOT NULL, + environment integer NOT NULL, + service integer NOT NULL, + route integer NOT NULL, + method smallint NOT NULL, + status smallint NOT NULL, + api_version integer NOT NULL, + source smallint NOT NULL, + platform smallint NOT NULL, + requests bigint NOT NULL, + errors bigint NOT NULL, + client_errors bigint NOT NULL, + sampled bigint NOT NULL, + duration_sum bigint NOT NULL, + duration_min bigint NOT NULL, + duration_max bigint NOT NULL, + durations bytea NOT NULL, + request_bytes bigint NOT NULL, + response_bytes bigint NOT NULL, + response_sizes bytea NOT NULL, + PRIMARY KEY (property_id, hour, environment, service, route, method, status, api_version, source, platform) +) PARTITION BY RANGE (hour); + +-- The extra dimensions (dim 1 consumer, 2 country, 3 app version) by local hour, each with route, +-- method and status class (0 for no response, else the status's hundreds). Written per writer like +-- the minute rows and compacted to writer 0 when the hour is folded. +CREATE TABLE monitor_dim_hour ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + hour timestamptz NOT NULL, + writer bigint NOT NULL, + dim smallint NOT NULL, + value integer NOT NULL, + route integer NOT NULL, + method smallint NOT NULL, + status_class smallint NOT NULL, + requests bigint NOT NULL, + errors bigint NOT NULL, + client_errors bigint NOT NULL, + sampled bigint NOT NULL, + duration_sum bigint NOT NULL, + duration_min bigint NOT NULL, + duration_max bigint NOT NULL, + durations bytea NOT NULL, + request_bytes bigint NOT NULL, + response_bytes bigint NOT NULL, + response_sizes bytea NOT NULL, + PRIMARY KEY (property_id, hour, writer, dim, value, route, method, status_class) +) PARTITION BY RANGE (hour); + +-- Local days folded from the hours, kept the property's retention_months. +CREATE TABLE monitor_day ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + day date NOT NULL, + environment integer NOT NULL, + service integer NOT NULL, + route integer NOT NULL, + method smallint NOT NULL, + status smallint NOT NULL, + api_version integer NOT NULL, + source smallint NOT NULL, + platform smallint NOT NULL, + requests bigint NOT NULL, + errors bigint NOT NULL, + client_errors bigint NOT NULL, + sampled bigint NOT NULL, + duration_sum bigint NOT NULL, + duration_min bigint NOT NULL, + duration_max bigint NOT NULL, + durations bytea NOT NULL, + request_bytes bigint NOT NULL, + response_bytes bigint NOT NULL, + response_sizes bytea NOT NULL, + PRIMARY KEY (property_id, day, environment, service, route, method, status, api_version, source, platform), + FOREIGN KEY (workspace_id, property_id) REFERENCES analytics_properties (workspace_id, id) ON DELETE CASCADE +); + +CREATE TABLE monitor_dim_day ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + day date NOT NULL, + dim smallint NOT NULL, + value integer NOT NULL, + route integer NOT NULL, + method smallint NOT NULL, + status_class smallint NOT NULL, + requests bigint NOT NULL, + errors bigint NOT NULL, + client_errors bigint NOT NULL, + sampled bigint NOT NULL, + duration_sum bigint NOT NULL, + duration_min bigint NOT NULL, + duration_max bigint NOT NULL, + durations bytea NOT NULL, + request_bytes bigint NOT NULL, + response_bytes bigint NOT NULL, + response_sizes bytea NOT NULL, + PRIMARY KEY (property_id, day, dim, value, route, method, status_class), + FOREIGN KEY (workspace_id, property_id) REFERENCES analytics_properties (workspace_id, id) ON DELETE CASCADE +); + +-- A row per property and local hour that has rows: built_at once the hour is folded, dirty when rows +-- arrived after that, capped when the hour reached its dimension rows, day_built_at once the hour's +-- local day was folded from it (cleared when the hour is folded again). +CREATE TABLE monitor_rollup_hours ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + hour timestamptz NOT NULL, + day date NOT NULL, + built_at timestamptz, + dirty boolean NOT NULL DEFAULT false, + capped boolean NOT NULL DEFAULT false, + day_built_at timestamptz, + PRIMARY KEY (property_id, hour), + FOREIGN KEY (workspace_id, property_id) REFERENCES analytics_properties (workspace_id, id) ON DELETE CASCADE +); +CREATE INDEX monitor_rollup_hours_todo_idx ON monitor_rollup_hours (workspace_id, property_id, hour) + WHERE built_at IS NULL OR dirty OR day_built_at IS NULL; +CREATE INDEX monitor_rollup_hours_day_idx ON monitor_rollup_hours (property_id, day); + +CREATE TABLE monitor_ingest_stats ( + workspace_id uuid NOT NULL, + property_id uuid NOT NULL, + day date NOT NULL, + reason text NOT NULL CHECK (reason IN ('accepted', 'invalid', 'late', 'rate', 'over_limit', 'buffer')), + records bigint NOT NULL, + PRIMARY KEY (property_id, day, reason), + FOREIGN KEY (workspace_id, property_id) REFERENCES analytics_properties (workspace_id, id) ON DELETE CASCADE +); + +-- Request records counted against YUVA_MONITOR_MONTHLY_REQUESTS, apart from analytics events. +ALTER TABLE usage_counters ADD COLUMN monitor_requests bigint NOT NULL DEFAULT 0; + +-- The operator's override of YUVA_MONITOR_MONTHLY_REQUESTS; 0 is no limit. +ALTER TABLE workspaces ADD COLUMN monitor_monthly_requests bigint CHECK (monitor_monthly_requests >= 0); + +-- +goose Down +ALTER TABLE workspaces DROP COLUMN monitor_monthly_requests; +ALTER TABLE usage_counters DROP COLUMN monitor_requests; +DROP TABLE monitor_ingest_stats; +DROP TABLE monitor_rollup_hours; +DROP TABLE monitor_dim_day; +DROP TABLE monitor_day; +DROP TABLE monitor_dim_hour; +DROP TABLE monitor_hour; +DROP TABLE monitor_minute; +DROP TABLE monitor_names; +DROP TABLE monitor_routes; +DELETE FROM analytics_properties WHERE kind = 'api'; +ALTER TABLE analytics_properties + DROP CONSTRAINT analytics_properties_api_check, + DROP CONSTRAINT analytics_properties_platforms_check, + ADD CONSTRAINT analytics_properties_platforms_check CHECK (cardinality(platforms) >= 1 AND platforms <@ ARRAY['web', 'ios', 'android']), + DROP CONSTRAINT analytics_properties_domains_check, + ADD CONSTRAINT analytics_properties_domains_check CHECK (cardinality(domains) BETWEEN 1 AND 50), + DROP COLUMN monitor_hour_days, + DROP COLUMN apdex_ms, + DROP COLUMN kind; diff --git a/api/internal/store/models.gen.go b/api/internal/store/models.gen.go index 80f3d4a9..58c7e58c 100644 --- a/api/internal/store/models.gen.go +++ b/api/internal/store/models.gen.go @@ -290,6 +290,9 @@ type AnalyticsProperty struct { DeletedAt *time.Time GoalsVersion int64 CohortsBuiltAt *time.Time + Kind string + ApdexMs int32 + MonitorHourDays int32 } type AnalyticsRollupDay struct { @@ -1348,6 +1351,167 @@ type MigrationUploadPart struct { CreatedAt time.Time } +type MonitorDay struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Day time.Time + Environment int32 + Service int32 + Route int32 + Method int16 + Status int16 + ApiVersion int32 + Source int16 + Platform int16 + Requests int64 + Errors int64 + ClientErrors int64 + Sampled int64 + DurationSum int64 + DurationMin int64 + DurationMax int64 + Durations []byte + RequestBytes int64 + ResponseBytes int64 + ResponseSizes []byte +} + +type MonitorDimDay struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Day time.Time + Dim int16 + Value int32 + Route int32 + Method int16 + StatusClass int16 + Requests int64 + Errors int64 + ClientErrors int64 + Sampled int64 + DurationSum int64 + DurationMin int64 + DurationMax int64 + Durations []byte + RequestBytes int64 + ResponseBytes int64 + ResponseSizes []byte +} + +type MonitorDimHour struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Hour time.Time + Writer int64 + Dim int16 + Value int32 + Route int32 + Method int16 + StatusClass int16 + Requests int64 + Errors int64 + ClientErrors int64 + Sampled int64 + DurationSum int64 + DurationMin int64 + DurationMax int64 + Durations []byte + RequestBytes int64 + ResponseBytes int64 + ResponseSizes []byte +} + +type MonitorHour struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Hour time.Time + Environment int32 + Service int32 + Route int32 + Method int16 + Status int16 + ApiVersion int32 + Source int16 + Platform int16 + Requests int64 + Errors int64 + ClientErrors int64 + Sampled int64 + DurationSum int64 + DurationMin int64 + DurationMax int64 + Durations []byte + RequestBytes int64 + ResponseBytes int64 + ResponseSizes []byte +} + +type MonitorIngestStat struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Day time.Time + Reason string + Records int64 +} + +type MonitorMinute struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Minute time.Time + Writer int64 + Environment int32 + Service int32 + Route int32 + Method int16 + Status int16 + ApiVersion int32 + Source int16 + Platform int16 + Requests int64 + Errors int64 + ClientErrors int64 + Sampled int64 + DurationSum int64 + DurationMin int64 + DurationMax int64 + Durations []byte + RequestBytes int64 + ResponseBytes int64 + ResponseSizes []byte +} + +type MonitorName struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Kind string + ID int32 + Value string + FirstSeenAt time.Time +} + +type MonitorRollupHour struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Hour time.Time + Day time.Time + BuiltAt *time.Time + Dirty bool + Capped bool + DayBuiltAt *time.Time +} + +type MonitorRoute struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + ID int32 + Pattern string + Inferred bool + Hidden bool + ApdexMs *int32 + FirstSeenAt time.Time + LastSeenAt time.Time +} + type NotificationEmail struct { WorkspaceID uuid.UUID MemberID uuid.UUID @@ -1531,6 +1695,7 @@ type UsageCounter struct { ImportedBytes int64 AnalyticsEvents int64 AnalyticsLimitReachedAt *time.Time + MonitorRequests int64 } type WebauthnCeremony struct { @@ -1604,4 +1769,5 @@ type Workspace struct { BroadcastDailyLimit *int32 BroadcastsByKeys bool AnalyticsMonthlyEvents *int64 + MonitorMonthlyRequests *int64 } diff --git a/api/internal/store/monitor.sql.gen.go b/api/internal/store/monitor.sql.gen.go new file mode 100644 index 00000000..b6a9646d --- /dev/null +++ b/api/internal/store/monitor.sql.gen.go @@ -0,0 +1,325 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: monitor.sql + +package store + +import ( + "context" + "time" + + "uuid" +) + +const deleteMonitorMinutes = `-- name: DeleteMonitorMinutes :execrows +DELETE FROM monitor_minute WHERE (tableoid, ctid) IN ( + SELECT t.tableoid, t.ctid FROM monitor_minute t + WHERE t.workspace_id = $1 AND t.property_id = $2 + AND ($3::timestamptz IS NULL OR t.minute >= $3::timestamptz) + AND ($4::timestamptz IS NULL OR t.minute < $4::timestamptz) + LIMIT $5::int) +` + +type DeleteMonitorMinutesParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + FromAt *time.Time + ToAt *time.Time + MaxRows int32 +} + +func (q *Queries) DeleteMonitorMinutes(ctx context.Context, arg DeleteMonitorMinutesParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteMonitorMinutes, + arg.WorkspaceID, + arg.PropertyID, + arg.FromAt, + arg.ToAt, + arg.MaxRows, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const deleteMonitorRange = `-- name: DeleteMonitorRange :exec +WITH h AS (DELETE FROM monitor_hour WHERE monitor_hour.workspace_id = $1 AND monitor_hour.property_id = $2 + AND monitor_hour.hour >= $5 AND monitor_hour.hour < $6), + dh AS (DELETE FROM monitor_dim_hour WHERE monitor_dim_hour.workspace_id = $1 AND monitor_dim_hour.property_id = $2 + AND monitor_dim_hour.hour >= $5 AND monitor_dim_hour.hour < $6), + d AS (DELETE FROM monitor_day WHERE monitor_day.workspace_id = $1 AND monitor_day.property_id = $2 + AND monitor_day.day BETWEEN $3 AND $4), + dd AS (DELETE FROM monitor_dim_day WHERE monitor_dim_day.workspace_id = $1 AND monitor_dim_day.property_id = $2 + AND monitor_dim_day.day BETWEEN $3 AND $4) +DELETE FROM monitor_rollup_hours WHERE monitor_rollup_hours.workspace_id = $1 AND monitor_rollup_hours.property_id = $2 + AND monitor_rollup_hours.day BETWEEN $3 AND $4 +` + +type DeleteMonitorRangeParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + FromDay time.Time + ToDay time.Time + FromAt time.Time + ToAt time.Time +} + +func (q *Queries) DeleteMonitorRange(ctx context.Context, arg DeleteMonitorRangeParams) error { + _, err := q.db.Exec(ctx, deleteMonitorRange, + arg.WorkspaceID, + arg.PropertyID, + arg.FromDay, + arg.ToDay, + arg.FromAt, + arg.ToAt, + ) + return err +} + +const deleteMonitorRollups = `-- name: DeleteMonitorRollups :exec +WITH d AS (DELETE FROM monitor_day WHERE monitor_day.workspace_id = $1 AND monitor_day.property_id = $2), + dd AS (DELETE FROM monitor_dim_day WHERE monitor_dim_day.workspace_id = $1 AND monitor_dim_day.property_id = $2), + h AS (DELETE FROM monitor_hour WHERE monitor_hour.workspace_id = $1 AND monitor_hour.property_id = $2), + dh AS (DELETE FROM monitor_dim_hour WHERE monitor_dim_hour.workspace_id = $1 AND monitor_dim_hour.property_id = $2) +DELETE FROM monitor_rollup_hours WHERE monitor_rollup_hours.workspace_id = $1 AND monitor_rollup_hours.property_id = $2 +` + +type DeleteMonitorRollupsParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID +} + +func (q *Queries) DeleteMonitorRollups(ctx context.Context, arg DeleteMonitorRollupsParams) error { + _, err := q.db.Exec(ctx, deleteMonitorRollups, arg.WorkspaceID, arg.PropertyID) + return err +} + +const deleteOldMonitorIngestStats = `-- name: DeleteOldMonitorIngestStats :execrows +DELETE FROM monitor_ingest_stats WHERE workspace_id = $1 AND day < $2 +` + +type DeleteOldMonitorIngestStatsParams struct { + WorkspaceID uuid.UUID + Before time.Time +} + +func (q *Queries) DeleteOldMonitorIngestStats(ctx context.Context, arg DeleteOldMonitorIngestStatsParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteOldMonitorIngestStats, arg.WorkspaceID, arg.Before) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getMonitorUsage = `-- name: GetMonitorUsage :one +SELECT w.monitor_monthly_requests AS monthly_limit, + coalesce(u.monitor_requests, 0)::bigint AS requests +FROM workspaces w LEFT JOIN usage_counters u ON u.workspace_id = w.id AND u.month = $1 +WHERE w.id = $2 +` + +type GetMonitorUsageParams struct { + Month time.Time + WorkspaceID uuid.UUID +} + +type GetMonitorUsageRow struct { + MonthlyLimit *int64 + Requests int64 +} + +func (q *Queries) GetMonitorUsage(ctx context.Context, arg GetMonitorUsageParams) (GetMonitorUsageRow, error) { + row := q.db.QueryRow(ctx, getMonitorUsage, arg.Month, arg.WorkspaceID) + var i GetMonitorUsageRow + err := row.Scan(&i.MonthlyLimit, &i.Requests) + return i, err +} + +const listMonitorRetention = `-- name: ListMonitorRetention :many +SELECT id, retention_months, monitor_hour_days, timezone FROM analytics_properties +WHERE workspace_id = $1 AND kind = 'api' AND deleted_at IS NULL +ORDER BY id +` + +type ListMonitorRetentionRow struct { + ID uuid.UUID + RetentionMonths *int32 + MonitorHourDays int32 + Timezone string +} + +func (q *Queries) ListMonitorRetention(ctx context.Context, workspaceID uuid.UUID) ([]ListMonitorRetentionRow, error) { + rows, err := q.db.Query(ctx, listMonitorRetention, workspaceID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListMonitorRetentionRow + for rows.Next() { + var i ListMonitorRetentionRow + if err := rows.Scan( + &i.ID, + &i.RetentionMonths, + &i.MonitorHourDays, + &i.Timezone, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMonitorRollupProperties = `-- name: ListMonitorRollupProperties :many +SELECT DISTINCT h.property_id FROM monitor_rollup_hours h +JOIN analytics_properties p ON p.workspace_id = h.workspace_id AND p.id = h.property_id +WHERE h.workspace_id = $1 AND p.deleted_at IS NULL AND h.hour < $2 + AND (h.built_at IS NULL OR h.dirty OR h.day_built_at IS NULL) +ORDER BY h.property_id +` + +type ListMonitorRollupPropertiesParams struct { + WorkspaceID uuid.UUID + Before time.Time +} + +func (q *Queries) ListMonitorRollupProperties(ctx context.Context, arg ListMonitorRollupPropertiesParams) ([]uuid.UUID, error) { + rows, err := q.db.Query(ctx, listMonitorRollupProperties, arg.WorkspaceID, arg.Before) + if err != nil { + return nil, err + } + defer rows.Close() + var items []uuid.UUID + for rows.Next() { + var property_id uuid.UUID + if err := rows.Scan(&property_id); err != nil { + return nil, err + } + items = append(items, property_id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setWorkspaceMonitorLimit = `-- name: SetWorkspaceMonitorLimit :execrows +UPDATE workspaces SET monitor_monthly_requests = $1 WHERE id = $2 AND deleted_at IS NULL +` + +type SetWorkspaceMonitorLimitParams struct { + MonthlyRequests *int64 + ID uuid.UUID +} + +func (q *Queries) SetWorkspaceMonitorLimit(ctx context.Context, arg SetWorkspaceMonitorLimitParams) (int64, error) { + result, err := q.db.Exec(ctx, setWorkspaceMonitorLimit, arg.MonthlyRequests, arg.ID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const trimMonitorDay = `-- name: TrimMonitorDay :execrows +DELETE FROM monitor_day WHERE ctid = ANY(ARRAY( + SELECT t.ctid FROM monitor_day t + WHERE t.workspace_id = $1 AND t.property_id = $2 AND t.day < $3 LIMIT 1000)) +` + +type TrimMonitorDayParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Before time.Time +} + +func (q *Queries) TrimMonitorDay(ctx context.Context, arg TrimMonitorDayParams) (int64, error) { + result, err := q.db.Exec(ctx, trimMonitorDay, arg.WorkspaceID, arg.PropertyID, arg.Before) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const trimMonitorDimDay = `-- name: TrimMonitorDimDay :execrows +DELETE FROM monitor_dim_day WHERE ctid = ANY(ARRAY( + SELECT t.ctid FROM monitor_dim_day t + WHERE t.workspace_id = $1 AND t.property_id = $2 AND t.day < $3 LIMIT 1000)) +` + +type TrimMonitorDimDayParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Before time.Time +} + +func (q *Queries) TrimMonitorDimDay(ctx context.Context, arg TrimMonitorDimDayParams) (int64, error) { + result, err := q.db.Exec(ctx, trimMonitorDimDay, arg.WorkspaceID, arg.PropertyID, arg.Before) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const trimMonitorDimHour = `-- name: TrimMonitorDimHour :execrows +DELETE FROM monitor_dim_hour WHERE (tableoid, ctid) IN ( + SELECT t.tableoid, t.ctid FROM monitor_dim_hour t + WHERE t.workspace_id = $1 AND t.property_id = $2 AND t.hour < $3 LIMIT 1000) +` + +type TrimMonitorDimHourParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Before time.Time +} + +func (q *Queries) TrimMonitorDimHour(ctx context.Context, arg TrimMonitorDimHourParams) (int64, error) { + result, err := q.db.Exec(ctx, trimMonitorDimHour, arg.WorkspaceID, arg.PropertyID, arg.Before) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const trimMonitorHour = `-- name: TrimMonitorHour :execrows +DELETE FROM monitor_hour WHERE (tableoid, ctid) IN ( + SELECT t.tableoid, t.ctid FROM monitor_hour t + WHERE t.workspace_id = $1 AND t.property_id = $2 AND t.hour < $3 LIMIT 1000) +` + +type TrimMonitorHourParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Before time.Time +} + +func (q *Queries) TrimMonitorHour(ctx context.Context, arg TrimMonitorHourParams) (int64, error) { + result, err := q.db.Exec(ctx, trimMonitorHour, arg.WorkspaceID, arg.PropertyID, arg.Before) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const trimMonitorRollupHours = `-- name: TrimMonitorRollupHours :execrows +DELETE FROM monitor_rollup_hours WHERE ctid = ANY(ARRAY( + SELECT t.ctid FROM monitor_rollup_hours t + WHERE t.workspace_id = $1 AND t.property_id = $2 AND t.hour < $3 LIMIT 1000)) +` + +type TrimMonitorRollupHoursParams struct { + WorkspaceID uuid.UUID + PropertyID uuid.UUID + Before time.Time +} + +func (q *Queries) TrimMonitorRollupHours(ctx context.Context, arg TrimMonitorRollupHoursParams) (int64, error) { + result, err := q.db.Exec(ctx, trimMonitorRollupHours, arg.WorkspaceID, arg.PropertyID, arg.Before) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/api/internal/store/queries/analytics.sql b/api/internal/store/queries/analytics.sql index 6007519e..82ec2cae 100644 --- a/api/internal/store/queries/analytics.sql +++ b/api/internal/store/queries/analytics.sql @@ -1,10 +1,10 @@ -- name: CreateAnalyticsProperty :one -INSERT INTO analytics_properties (id, workspace_id, inbox_id, name, public_key, domains, platforms, timezone, currency, collection, - consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, visitor_key, created_by, - created_at, updated_at) -VALUES (@id, @workspace_id, sqlc.narg(inbox_id), @name, @public_key, @domains, @platforms, @timezone, @currency, @collection, - @consent_features, @audience, @excluded_paths, @excluded_ips, @path_rules, @tracked_props, sqlc.narg(retention_months), @visitor_key, - sqlc.narg(created_by), @now, @now) +INSERT INTO analytics_properties (id, workspace_id, inbox_id, kind, name, public_key, domains, platforms, timezone, currency, collection, + consent_features, audience, excluded_paths, excluded_ips, path_rules, tracked_props, retention_months, apdex_ms, monitor_hour_days, + visitor_key, created_by, created_at, updated_at) +VALUES (@id, @workspace_id, sqlc.narg(inbox_id), @kind, @name, @public_key, @domains, @platforms, @timezone, @currency, @collection, + @consent_features, @audience, @excluded_paths, @excluded_ips, @path_rules, @tracked_props, sqlc.narg(retention_months), @apdex_ms, + @monitor_hour_days, @visitor_key, sqlc.narg(created_by), @now, @now) RETURNING *; -- name: GetAnalyticsProperty :one @@ -29,7 +29,7 @@ UPDATE analytics_properties SET inbox_id = sqlc.narg(inbox_id), name = @name, domains = @domains, platforms = @platforms, timezone = @timezone, currency = @currency, collection = @collection, consent_features = @consent_features, audience = @audience, excluded_paths = @excluded_paths, excluded_ips = @excluded_ips, path_rules = @path_rules, tracked_props = @tracked_props, - retention_months = sqlc.narg(retention_months), updated_at = @now + retention_months = sqlc.narg(retention_months), apdex_ms = @apdex_ms, monitor_hour_days = @monitor_hour_days, updated_at = @now WHERE workspace_id = @workspace_id AND id = @id AND deleted_at IS NULL RETURNING *; diff --git a/api/internal/store/queries/monitor.sql b/api/internal/store/queries/monitor.sql new file mode 100644 index 00000000..2bb9f346 --- /dev/null +++ b/api/internal/store/queries/monitor.sql @@ -0,0 +1,75 @@ +-- name: GetMonitorUsage :one +SELECT w.monitor_monthly_requests AS monthly_limit, + coalesce(u.monitor_requests, 0)::bigint AS requests +FROM workspaces w LEFT JOIN usage_counters u ON u.workspace_id = w.id AND u.month = @month +WHERE w.id = @workspace_id; + +-- name: SetWorkspaceMonitorLimit :execrows +UPDATE workspaces SET monitor_monthly_requests = sqlc.narg(monthly_requests) WHERE id = @id AND deleted_at IS NULL; + +-- name: ListMonitorRollupProperties :many +SELECT DISTINCT h.property_id FROM monitor_rollup_hours h +JOIN analytics_properties p ON p.workspace_id = h.workspace_id AND p.id = h.property_id +WHERE h.workspace_id = @workspace_id AND p.deleted_at IS NULL AND h.hour < @before + AND (h.built_at IS NULL OR h.dirty OR h.day_built_at IS NULL) +ORDER BY h.property_id; + +-- name: ListMonitorRetention :many +SELECT id, retention_months, monitor_hour_days, timezone FROM analytics_properties +WHERE workspace_id = @workspace_id AND kind = 'api' AND deleted_at IS NULL +ORDER BY id; + +-- name: TrimMonitorDay :execrows +DELETE FROM monitor_day WHERE ctid = ANY(ARRAY( + SELECT t.ctid FROM monitor_day t + WHERE t.workspace_id = @workspace_id AND t.property_id = @property_id AND t.day < @before LIMIT 1000)); + +-- name: TrimMonitorDimDay :execrows +DELETE FROM monitor_dim_day WHERE ctid = ANY(ARRAY( + SELECT t.ctid FROM monitor_dim_day t + WHERE t.workspace_id = @workspace_id AND t.property_id = @property_id AND t.day < @before LIMIT 1000)); + +-- name: TrimMonitorHour :execrows +DELETE FROM monitor_hour WHERE (tableoid, ctid) IN ( + SELECT t.tableoid, t.ctid FROM monitor_hour t + WHERE t.workspace_id = @workspace_id AND t.property_id = @property_id AND t.hour < @before LIMIT 1000); + +-- name: TrimMonitorDimHour :execrows +DELETE FROM monitor_dim_hour WHERE (tableoid, ctid) IN ( + SELECT t.tableoid, t.ctid FROM monitor_dim_hour t + WHERE t.workspace_id = @workspace_id AND t.property_id = @property_id AND t.hour < @before LIMIT 1000); + +-- name: TrimMonitorRollupHours :execrows +DELETE FROM monitor_rollup_hours WHERE ctid = ANY(ARRAY( + SELECT t.ctid FROM monitor_rollup_hours t + WHERE t.workspace_id = @workspace_id AND t.property_id = @property_id AND t.hour < @before LIMIT 1000)); + +-- name: DeleteOldMonitorIngestStats :execrows +DELETE FROM monitor_ingest_stats WHERE workspace_id = @workspace_id AND day < @before; + +-- name: DeleteMonitorRollups :exec +WITH d AS (DELETE FROM monitor_day WHERE monitor_day.workspace_id = @workspace_id AND monitor_day.property_id = @property_id), + dd AS (DELETE FROM monitor_dim_day WHERE monitor_dim_day.workspace_id = @workspace_id AND monitor_dim_day.property_id = @property_id), + h AS (DELETE FROM monitor_hour WHERE monitor_hour.workspace_id = @workspace_id AND monitor_hour.property_id = @property_id), + dh AS (DELETE FROM monitor_dim_hour WHERE monitor_dim_hour.workspace_id = @workspace_id AND monitor_dim_hour.property_id = @property_id) +DELETE FROM monitor_rollup_hours WHERE monitor_rollup_hours.workspace_id = @workspace_id AND monitor_rollup_hours.property_id = @property_id; + +-- name: DeleteMonitorMinutes :execrows +DELETE FROM monitor_minute WHERE (tableoid, ctid) IN ( + SELECT t.tableoid, t.ctid FROM monitor_minute t + WHERE t.workspace_id = @workspace_id AND t.property_id = @property_id + AND (sqlc.narg(from_at)::timestamptz IS NULL OR t.minute >= sqlc.narg(from_at)::timestamptz) + AND (sqlc.narg(to_at)::timestamptz IS NULL OR t.minute < sqlc.narg(to_at)::timestamptz) + LIMIT @max_rows::int); + +-- name: DeleteMonitorRange :exec +WITH h AS (DELETE FROM monitor_hour WHERE monitor_hour.workspace_id = @workspace_id AND monitor_hour.property_id = @property_id + AND monitor_hour.hour >= @from_at AND monitor_hour.hour < @to_at), + dh AS (DELETE FROM monitor_dim_hour WHERE monitor_dim_hour.workspace_id = @workspace_id AND monitor_dim_hour.property_id = @property_id + AND monitor_dim_hour.hour >= @from_at AND monitor_dim_hour.hour < @to_at), + d AS (DELETE FROM monitor_day WHERE monitor_day.workspace_id = @workspace_id AND monitor_day.property_id = @property_id + AND monitor_day.day BETWEEN @from_day AND @to_day), + dd AS (DELETE FROM monitor_dim_day WHERE monitor_dim_day.workspace_id = @workspace_id AND monitor_dim_day.property_id = @property_id + AND monitor_dim_day.day BETWEEN @from_day AND @to_day) +DELETE FROM monitor_rollup_hours WHERE monitor_rollup_hours.workspace_id = @workspace_id AND monitor_rollup_hours.property_id = @property_id + AND monitor_rollup_hours.day BETWEEN @from_day AND @to_day; diff --git a/api/internal/store/usage.sql.gen.go b/api/internal/store/usage.sql.gen.go index 263c0144..c71d7ac4 100644 --- a/api/internal/store/usage.sql.gen.go +++ b/api/internal/store/usage.sql.gen.go @@ -69,7 +69,7 @@ func (q *Queries) AddUsage(ctx context.Context, arg AddUsageParams) error { } const listUsage = `-- name: ListUsage :many -SELECT workspace_id, month, conversations, messages, attachment_bytes, broadcast_recipients, imported_conversations, imported_messages, imported_bytes, analytics_events, analytics_limit_reached_at FROM usage_counters WHERE workspace_id = $1 ORDER BY month DESC LIMIT 24 +SELECT workspace_id, month, conversations, messages, attachment_bytes, broadcast_recipients, imported_conversations, imported_messages, imported_bytes, analytics_events, analytics_limit_reached_at, monitor_requests FROM usage_counters WHERE workspace_id = $1 ORDER BY month DESC LIMIT 24 ` func (q *Queries) ListUsage(ctx context.Context, workspaceID uuid.UUID) ([]UsageCounter, error) { @@ -93,6 +93,7 @@ func (q *Queries) ListUsage(ctx context.Context, workspaceID uuid.UUID) ([]Usage &i.ImportedBytes, &i.AnalyticsEvents, &i.AnalyticsLimitReachedAt, + &i.MonitorRequests, ); err != nil { return nil, err } diff --git a/api/internal/store/workspaces.sql.gen.go b/api/internal/store/workspaces.sql.gen.go index 4619e00c..b8edfdbb 100644 --- a/api/internal/store/workspaces.sql.gen.go +++ b/api/internal/store/workspaces.sql.gen.go @@ -64,7 +64,7 @@ func (q *Queries) CountWorkspacesCreatedFromIP(ctx context.Context, arg CountWor const createWorkspace = `-- name: CreateWorkspace :one INSERT INTO workspaces (id, name, created_by, created_ip) VALUES ($1, $2, $3, $4) -RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events +RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests ` type CreateWorkspaceParams struct { @@ -97,6 +97,7 @@ func (q *Queries) CreateWorkspace(ctx context.Context, arg CreateWorkspaceParams &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } @@ -151,7 +152,7 @@ func (q *Queries) DeleteWorkspaceWebhookEndpoints(ctx context.Context, workspace } const getDeletedWorkspace = `-- name: GetDeletedWorkspace :one -SELECT id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events FROM workspaces WHERE id = $1 AND deleted_at IS NOT NULL +SELECT id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests FROM workspaces WHERE id = $1 AND deleted_at IS NOT NULL ` func (q *Queries) GetDeletedWorkspace(ctx context.Context, id uuid.UUID) (Workspace, error) { @@ -172,12 +173,13 @@ func (q *Queries) GetDeletedWorkspace(ctx context.Context, id uuid.UUID) (Worksp &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } const getWorkspace = `-- name: GetWorkspace :one -SELECT id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events FROM workspaces WHERE id = $1 +SELECT id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests FROM workspaces WHERE id = $1 ` func (q *Queries) GetWorkspace(ctx context.Context, id uuid.UUID) (Workspace, error) { @@ -198,6 +200,7 @@ func (q *Queries) GetWorkspace(ctx context.Context, id uuid.UUID) (Workspace, er &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } @@ -351,7 +354,7 @@ func (q *Queries) ListWorkspaceRetention(ctx context.Context) ([]ListWorkspaceRe } const listWorkspacesByName = `-- name: ListWorkspacesByName :many -SELECT id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events FROM workspaces WHERE name = $1 AND deleted_at IS NULL ORDER BY created_at, id +SELECT id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests FROM workspaces WHERE name = $1 AND deleted_at IS NULL ORDER BY created_at, id ` func (q *Queries) ListWorkspacesByName(ctx context.Context, name string) ([]Workspace, error) { @@ -378,6 +381,7 @@ func (q *Queries) ListWorkspacesByName(ctx context.Context, name string) ([]Work &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ); err != nil { return nil, err } @@ -441,7 +445,7 @@ func (q *Queries) PurgeWorkspace(ctx context.Context, id uuid.UUID) (int64, erro const setWorkspaceBotsMaySend = `-- name: SetWorkspaceBotsMaySend :one UPDATE workspaces SET bots_may_send = $1 WHERE id = $2 -RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events +RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests ` type SetWorkspaceBotsMaySendParams struct { @@ -467,6 +471,7 @@ func (q *Queries) SetWorkspaceBotsMaySend(ctx context.Context, arg SetWorkspaceB &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } @@ -474,7 +479,7 @@ func (q *Queries) SetWorkspaceBotsMaySend(ctx context.Context, arg SetWorkspaceB const setWorkspaceRetention = `-- name: SetWorkspaceRetention :one UPDATE workspaces SET retention_days = $1 WHERE id = $2 -RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events +RETURNING id, name, created_at, retention_days, deleted_at, bots_may_send, created_by, created_ip, broadcasts_status, broadcasts_status_reason, broadcasts_status_at, broadcast_daily_limit, broadcasts_by_keys, analytics_monthly_events, monitor_monthly_requests ` type SetWorkspaceRetentionParams struct { @@ -500,6 +505,7 @@ func (q *Queries) SetWorkspaceRetention(ctx context.Context, arg SetWorkspaceRet &i.BroadcastDailyLimit, &i.BroadcastsByKeys, &i.AnalyticsMonthlyEvents, + &i.MonitorMonthlyRequests, ) return i, err } diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 0ad1d2e6..5c50cf2a 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -243,6 +243,12 @@ tags: the address, and host backends send server-side events with an API key that has `analytics:write` (`sendAnalyticsEvents`). + A property of `kind: api` watches an API instead of a website: its host reports one record + per request it served (`sendAnalyticsRequests`), and `queryAnalytics` with `dataset: api` + counts requests, errors and latency by route. Records are added up per minute and hour as + they arrive; no row per request and no address is kept. `YUVA_MONITOR=off` removes API + properties and their endpoints. + Owners and admins manage properties. Agents read the properties linked to inboxes they can access; a property without an inbox is visible to owners, admins and API keys without an inbox limit. Properties the caller cannot see answer `404`. Keys and OAuth tokens need @@ -7678,11 +7684,12 @@ paths: operationId: createAnalyticsProperty summary: Create an analytics property description: | - A website or app to measure. The answer carries its `public_key`, which pages and apps send - events with. `inbox_id` links the property to an inbox: agents of that inbox can then read - it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties - (`403 property_limit`). A children's property (`audience: children`) cannot offer - `consent_features` or `tracked_props`. + A website or app to measure, or with `kind: api` an API to watch. The answer carries its + `public_key`, which pages and apps send events with. `inbox_id` links the property to an + inbox: agents of that inbox can then read it. A workspace has at most + `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's + property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API + property offers neither, and `audience` does not apply to it. Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. tags: [analytics] @@ -7935,7 +7942,8 @@ paths: description: | `needs_raw_window`: the query needs raw rows for days that are no longer kept; `raw_window_from` is the first day it can serve. `range_too_large`: the range is longer - than 400 days. + than 400 days, or with `time:minute` more than 1,440 minutes or older than the minute + rows. `property_kind`: the dataset does not fit the property's kind. content: application/problem+json: schema: @@ -8034,6 +8042,74 @@ paths: $ref: "#/components/responses/Problem" "503": $ref: "#/components/responses/Problem" + /v1/analytics/properties/{propertyId}/requests: + post: + operationId: sendAnalyticsRequests + summary: Send an API's request records + description: | + The host of an API property reports the requests it served, 1 to 1,000 records a call, + with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with + `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key + limited to inboxes reaches the properties of its inboxes. A site answers + `422 property_kind`. Scope: `analytics:write`. + + A record names its route pattern (`/users/{id}`), as the router matched it. A record + that can only give its `path` gets a pattern inferred from it: the query and fragment are + cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other + tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both + cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that + answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The + property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 + routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), + environments (20) and services (100 per workspace) work the same way. + + Nothing else of a request is accepted: no body, header, query string, fragment or user + agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country + right away and never kept. Records are added up per minute and hour in the server's + memory and written every 10 seconds, so a request shows in `queryAnalytics` + (`dataset: api`) within about 15 seconds. + + Invalid records are left out on their own and listed in `errors`; the others are taken. + Taken but not counted: records older than 72 hours, records over the property's + `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap + (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may + (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. + tags: [analytics] + security: + - apiKey: [] + parameters: + - $ref: "#/components/parameters/AnalyticsPropertyId" + - name: Content-Encoding + in: header + description: "`gzip` when the body is compressed." + schema: + type: string + enum: [gzip, identity] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AnalyticsRequestRecords" + responses: + "202": + description: Taken; counted unless one of the rules above dropped them. + content: + application/json: + schema: + $ref: "#/components/schemas/AnalyticsRequestsResult" + "400": + $ref: "#/components/responses/Problem" + "401": + $ref: "#/components/responses/Problem" + "403": + $ref: "#/components/responses/Problem" + "404": + $ref: "#/components/responses/Problem" + "422": + $ref: "#/components/responses/Problem" + "503": + $ref: "#/components/responses/Problem" /v1/analytics/properties/{propertyId}/retention: get: operationId: getAnalyticsRetention @@ -12229,8 +12305,8 @@ components: Yuva. Owners and admins only; never offered to agents' tokens, refused for keys limited to inboxes. - `analytics:read`: analytics properties. - - `analytics:write`: server-side analytics events; for API keys only, never offered to - OAuth clients. + - `analytics:write`: server-side analytics events and API request records; for API keys + only, never offered to OAuth clients. - `analytics:manage`: create, change and delete analytics properties, rotate their keys, delete their data. Never offered to agents' tokens, refused for keys limited to inboxes. @@ -20523,15 +20599,27 @@ components: type: string minLength: 1 maxLength: 300 + AnalyticsPropertyKind: + type: string + description: | + `site` measures websites and apps (page views, events, visitors). `api` watches an API: its + host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An + API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps + allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, + tracked props or consent features. The kind is fixed when the property is made. + enum: [site, api] + x-enum-varnames: [AnalyticsKindSite, AnalyticsKindApi] AnalyticsProperty: type: object - description: A website or app the workspace measures. - required: [id, name, public_key, domains, platforms, timezone, currency, collection, consent_features, audience, - excluded_paths, excluded_ips, path_rules, tracked_props, visitors_today, created_at, updated_at] + description: A website, app or API the workspace measures. + required: [id, kind, name, public_key, domains, platforms, timezone, currency, collection, consent_features, audience, + excluded_paths, excluded_ips, path_rules, tracked_props, apdex_ms, monitor_hour_days, visitors_today, created_at, updated_at] properties: id: type: string format: uuid + kind: + $ref: "#/components/schemas/AnalyticsPropertyKind" inbox_id: type: string format: uuid @@ -20589,6 +20677,14 @@ components: type: integer format: int32 description: Rollups older than this are deleted; absent when they are kept forever. + apdex_ms: + type: integer + format: int32 + description: An API's Apdex threshold T in milliseconds; a request is satisfied within T and tolerated within 4T. + monitor_hour_days: + type: integer + format: int32 + description: How many days an API's hourly rows are kept; daily rows follow `retention_months`. visitors_today: type: integer format: int64 @@ -20611,6 +20707,8 @@ components: type: object required: [name, domains] properties: + kind: + $ref: "#/components/schemas/AnalyticsPropertyKind" name: type: string minLength: 1 @@ -20620,16 +20718,27 @@ components: format: uuid domains: type: array - minItems: 1 + description: 1 to 50 host names for a site, 0 to 50 for an API. maxItems: 50 items: $ref: "#/components/schemas/AnalyticsDomain" platforms: type: array - description: "`[web]` when absent." - minItems: 1 + description: "At least one for a site, `[web]` when absent; for an API `ios` and `android` only, none when absent." items: $ref: "#/components/schemas/AnalyticsPlatform" + apdex_ms: + type: integer + format: int32 + minimum: 1 + maximum: 60000 + default: 500 + monitor_hour_days: + type: integer + format: int32 + minimum: 7 + maximum: 400 + default: 90 timezone: type: string description: The linked inbox's time zone when absent, else `UTC`. @@ -20685,15 +20794,25 @@ components: description: "`null` unlinks the property from its inbox." domains: type: array - minItems: 1 + description: 1 to 50 host names for a site, 0 to 50 for an API. maxItems: 50 items: $ref: "#/components/schemas/AnalyticsDomain" platforms: type: array - minItems: 1 + description: At least one for a site; for an API `ios` and `android` only. items: $ref: "#/components/schemas/AnalyticsPlatform" + apdex_ms: + type: integer + format: int32 + minimum: 1 + maximum: 60000 + monitor_hour_days: + type: integer + format: int32 + minimum: 7 + maximum: 400 timezone: type: string currency: @@ -20754,6 +20873,14 @@ components: type: string format: date-time description: When this month's events reached the cap; later events are dropped until the month ends. + requests: + type: integer + format: int64 + description: API request records counted this month (`sendAnalyticsRequests`); absent when API monitoring is off. + requests_limit: + type: integer + format: int64 + description: The monthly cap on request records; absent when there is none. Records over it are not counted until the month ends. AnalyticsPropertyList: type: object required: [items, usage] @@ -20806,13 +20933,23 @@ components: `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), `discussions` (discussion threads started on it) and `support_rate` (conversations per 1,000 page views), next to `visitors` and `pageviews`. + + With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests + per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no + response was sent), `error_rate` and `client_error_rate` (percent of requests), + `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds + that this share of requests stayed within, from sketches accurate to 1.1%) and + `avg_duration` (milliseconds). enum: [visitors, visits, pageviews, views_per_visit, bounce_rate, visit_duration, time_on_page, scroll_depth, events, conversions, - conversion_rate, completions, revenue, conversations, feedback, ratings_up, ratings_down, helpful_rate, discussions, support_rate] + conversion_rate, completions, revenue, conversations, feedback, ratings_up, ratings_down, helpful_rate, discussions, support_rate, + requests, requests_per_minute, errors, error_rate, client_errors, client_error_rate, p50, p95, p99, avg_duration] x-enum-varnames: [AnalyticsMetricVisitors, AnalyticsMetricVisits, AnalyticsMetricPageviews, AnalyticsMetricViewsPerVisit, AnalyticsMetricBounceRate, AnalyticsMetricVisitDuration, AnalyticsMetricTimeOnPage, AnalyticsMetricScrollDepth, AnalyticsMetricEvents, AnalyticsMetricConversions, AnalyticsMetricConversionRate, AnalyticsMetricCompletions, AnalyticsMetricRevenue, AnalyticsMetricConversations, AnalyticsMetricFeedback, AnalyticsMetricRatingsUp, AnalyticsMetricRatingsDown, AnalyticsMetricHelpfulRate, - AnalyticsMetricDiscussions, AnalyticsMetricSupportRate] + AnalyticsMetricDiscussions, AnalyticsMetricSupportRate, AnalyticsMetricRequests, AnalyticsMetricRequestsPerMinute, AnalyticsMetricErrors, + AnalyticsMetricErrorRate, AnalyticsMetricClientErrors, AnalyticsMetricClientErrorRate, AnalyticsMetricP50, AnalyticsMetricP95, + AnalyticsMetricP99, AnalyticsMetricAvgDuration] AnalyticsFilterDimension: type: string description: | @@ -20832,14 +20969,21 @@ components: for events without it: as a filter of visits they keep the visits with such an event; in an `event` or `prop` breakdown they keep the matching events. + + With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for + requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 + routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response + was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's + environment (`production` by default, `(other)` past 20). enum: [page, entry_page, exit_page, hostname, channel, source, referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, country, region, city, device, browser, browser_version, os, os_version, screen_size, language, platform, app_version, - visitor_type, event, prop] + visitor_type, event, prop, route, method, status, status_class, environment] x-enum-varnames: [AnalyticsDimPage, AnalyticsDimEntryPage, AnalyticsDimExitPage, AnalyticsDimHostname, AnalyticsDimChannel, AnalyticsDimSource, AnalyticsDimReferrer, AnalyticsDimUtmSource, AnalyticsDimUtmMedium, AnalyticsDimUtmCampaign, AnalyticsDimUtmTerm, AnalyticsDimUtmContent, AnalyticsDimCountry, AnalyticsDimRegion, AnalyticsDimCity, AnalyticsDimDevice, AnalyticsDimBrowser, AnalyticsDimBrowserVersion, AnalyticsDimOs, AnalyticsDimOsVersion, AnalyticsDimScreenSize, AnalyticsDimLanguage, AnalyticsDimPlatform, - AnalyticsDimAppVersion, AnalyticsDimVisitorType, AnalyticsDimEvent, AnalyticsDimProp] + AnalyticsDimAppVersion, AnalyticsDimVisitorType, AnalyticsDimEvent, AnalyticsDimProp, AnalyticsDimRoute, AnalyticsDimMethod, + AnalyticsDimStatus, AnalyticsDimStatusClass, AnalyticsDimEnvironment] AnalyticsDimension: type: string description: | @@ -20850,14 +20994,20 @@ components: range is present. `event` breaks custom and built-in events down by name, `prop` by the value of the event prop the query's `prop` names; either takes at most one other dimension, a time grain other than `time:hour` or a visit dimension. - enum: [time:hour, time:day, time:week, time:month, page, entry_page, exit_page, hostname, channel, source, referrer, utm_source, + + With `dataset: api`, two of `route`, `method`, `status`, `status_class`, `environment` and + one grain: `time:minute` (minutes, `2026-10-12T14:05` in the property's time zone, inside + the last `YUVA_MONITOR_MINUTE_DAYS` days and at most 1,440 of them), `time:hour` (local + hours) or `time:day`. + enum: [time:minute, time:hour, time:day, time:week, time:month, page, entry_page, exit_page, hostname, channel, source, referrer, utm_source, utm_medium, utm_campaign, utm_term, utm_content, country, region, city, device, browser, browser_version, os, os_version, - screen_size, language, platform, app_version, visitor_type, event, prop] - x-enum-varnames: [AnalyticsByHour, AnalyticsByDay, AnalyticsByWeek, AnalyticsByMonth, AnalyticsByPage, AnalyticsByEntryPage, + screen_size, language, platform, app_version, visitor_type, event, prop, route, method, status, status_class, environment] + x-enum-varnames: [AnalyticsByMinute, AnalyticsByHour, AnalyticsByDay, AnalyticsByWeek, AnalyticsByMonth, AnalyticsByPage, AnalyticsByEntryPage, AnalyticsByExitPage, AnalyticsByHostname, AnalyticsByChannel, AnalyticsBySource, AnalyticsByReferrer, AnalyticsByUtmSource, AnalyticsByUtmMedium, AnalyticsByUtmCampaign, AnalyticsByUtmTerm, AnalyticsByUtmContent, AnalyticsByCountry, AnalyticsByRegion, AnalyticsByCity, AnalyticsByDevice, AnalyticsByBrowser, AnalyticsByBrowserVersion, AnalyticsByOs, AnalyticsByOsVersion, - AnalyticsByScreenSize, AnalyticsByLanguage, AnalyticsByPlatform, AnalyticsByAppVersion, AnalyticsByVisitorType, AnalyticsByEvent, AnalyticsByProp] + AnalyticsByScreenSize, AnalyticsByLanguage, AnalyticsByPlatform, AnalyticsByAppVersion, AnalyticsByVisitorType, AnalyticsByEvent, AnalyticsByProp, + AnalyticsByRoute, AnalyticsByMethod, AnalyticsByStatus, AnalyticsByStatusClass, AnalyticsByEnvironment] AnalyticsFilter: type: object description: | @@ -20891,13 +21041,15 @@ components: Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and `year` are the current calendar month and year to date; `last_month` the one before), or - `from` and `to` (inclusive, at most 400 days apart). + `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also + be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. properties: period: type: string - enum: [today, yesterday, 7d, 28d, 30d, 90d, 6mo, 12mo, month, last_month, year] + enum: [today, yesterday, 7d, 28d, 30d, 90d, 6mo, 12mo, month, last_month, year, 15m, 1h, 6h, 24h] x-enum-varnames: [AnalyticsToday, AnalyticsYesterday, AnalyticsLast7Days, AnalyticsLast28Days, AnalyticsLast30Days, - AnalyticsLast90Days, AnalyticsLast6Months, AnalyticsLast12Months, AnalyticsThisMonth, AnalyticsLastMonth, AnalyticsThisYear] + AnalyticsLast90Days, AnalyticsLast6Months, AnalyticsLast12Months, AnalyticsThisMonth, AnalyticsLastMonth, AnalyticsThisYear, + AnalyticsLast15Minutes, AnalyticsLastHour, AnalyticsLast6Hours, AnalyticsLast24Hours] from: type: string format: date @@ -20946,8 +21098,16 @@ components: `support_rate` with pages of fewer than 100 page views after the others unless `order_by` says otherwise. Days of the past come from the support rollup, today and days not yet rolled up from the inbox itself. - enum: [traffic, support] - x-enum-varnames: [AnalyticsDatasetTraffic, AnalyticsDatasetSupport] + + `api` is an API property's requests (`sendAnalyticsRequests`), and the only dataset an API + property takes (`422 property_kind` otherwise, and for `api` on a site). It takes the API + metrics of `AnalyticsMetric`, the API dimensions of `AnalyticsDimension` and filters on + them, no goal, prop, segment, comparison or insights. Closed local days come from daily + rows, closed hours from hourly rows and the rest from minute rows, which processes write + every 10 seconds; `meta.served_from` is `rollups` when only daily and hourly rows answered. + Rows are ordered by `requests` unless `order_by` says otherwise. + enum: [traffic, support, api] + x-enum-varnames: [AnalyticsDatasetTraffic, AnalyticsDatasetSupport, AnalyticsDatasetApi] AnalyticsQuery: type: object required: [metrics, range] @@ -21077,6 +21237,36 @@ components: support_rate: type: number format: double + requests: + type: number + format: double + requests_per_minute: + type: number + format: double + errors: + type: number + format: double + error_rate: + type: number + format: double + client_errors: + type: number + format: double + client_error_rate: + type: number + format: double + p50: + type: number + format: double + p95: + type: number + format: double + p99: + type: number + format: double + avg_duration: + type: number + format: double AnalyticsRevenue: type: object required: [currency, total, average, events, digits] @@ -21153,6 +21343,21 @@ components: type: string format: date description: The first day raw rows are kept for, which filters and breakdowns the rollups cannot answer need. + estimated: + type: boolean + description: "`dataset: api`: some requests of the range were sampled (`sample_rate`), so counts are estimates." + caps_reached: + type: array + description: | + `dataset: api`: the caps the property has reached, so new values count as `(other)`: + `routes` (1,000), `consumers` (10,000), `api_versions` (50), `app_versions` (200), + `environments` (20), `services` (100 per workspace), `dimension_rows` (an hour of the + range reached 50,000 consumer, country or app version rows). + items: + type: string + enum: [routes, consumers, api_versions, app_versions, environments, services, dimension_rows] + x-enum-varnames: [AnalyticsCapRoutes, AnalyticsCapConsumers, AnalyticsCapApiVersions, AnalyticsCapAppVersions, + AnalyticsCapEnvironments, AnalyticsCapServices, AnalyticsCapDimensionRows] AnalyticsResult: type: object required: [rows, totals, meta] @@ -21465,6 +21670,108 @@ components: type: string maxLength: 8192 description: "An identity token as in `AnalyticsConsent.j`, with `persistent_id` and `consent: true`, on a property that offers `identify`." + AnalyticsRequestRecords: + type: object + required: [records] + properties: + records: + type: array + minItems: 1 + maxItems: 1000 + items: + $ref: "#/components/schemas/AnalyticsRequestRecord" + AnalyticsRequestRecord: + type: object + description: One request an API served. Give `route`, or `path` when the router knows no pattern. + required: [method, status, duration_us] + additionalProperties: false + properties: + method: + type: string + minLength: 1 + maxLength: 20 + description: The request method; anything but the RFC 9110 methods and `PATCH` counts as `_OTHER`. + examples: [GET] + route: + type: string + minLength: 1 + maxLength: 300 + description: The route pattern that matched, starting with `/`, without method or host. + examples: ["/users/{id}"] + path: + type: string + minLength: 1 + maxLength: 2000 + description: The request path, when there is no `route`; only the pattern inferred from it is kept. + examples: ["/users/42"] + status: + type: integer + minimum: 0 + maximum: 599 + description: The response status, 100 to 599, or 0 when no response was sent. + duration_us: + type: integer + format: int64 + minimum: 0 + description: How long the request took, in microseconds. + request_size: + type: integer + format: int64 + minimum: 0 + description: Bytes of the request body. + response_size: + type: integer + format: int64 + minimum: 0 + description: Bytes of the response body. + time: + type: string + format: date-time + description: When the request started; now when absent. Records older than 72 hours are not counted. + consumer: + type: string + minLength: 1 + maxLength: 100 + description: Who called, as you name them, such as an API key's name or a client id; never a person's address (e-mail-shaped values become `(scrubbed)`). + api_version: + type: string + maxLength: 40 + environment: + type: string + maxLength: 40 + description: "`production` when absent." + service: + type: string + maxLength: 100 + description: The service that served the request. + release: + type: string + maxLength: 80 + description: The service's release, for deploy markers. + ip: + type: string + maxLength: 100 + description: The caller's address, only to find its country; shortened on arrival, never stored. + sample_rate: + type: integer + minimum: 1 + maximum: 10000 + default: 1 + description: | + This record stands for this many requests, for a host that sends one in N successful + requests. Only records with a status of 1 to 399 may be sampled. + AnalyticsRequestsResult: + type: object + required: [accepted, errors] + properties: + accepted: + type: integer + description: The valid records, counted unless a rule of the operation dropped them. + errors: + type: array + description: The records left out as invalid, by their position in `records`. + items: + $ref: "#/components/schemas/AnalyticsEventError" AnalyticsEventsResult: type: object required: [accepted, errors] From 4fcac89e120b52938d69ddc18c2c1e8dec3f799c Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:40:13 +0200 Subject: [PATCH 03/69] feat(sdk/go): analytics Middleware records a net/http API's requests by the mux's route Middleware reads r.Pattern after the ServeMux ran (method and host left out) or a Route function, and sends route, method, status, duration and sizes in gzip batches of 500 every 5 seconds through the client's queue and retries; SampleRate thins successful requests only. The API client is regenerated for the requests endpoint. Co-Authored-By: Claude Opus 5.5 --- sdk/go/README.md | 36 + sdk/go/analytics/analytics.go | 79 ++- sdk/go/analytics/monitor.go | 313 +++++++++ sdk/go/analytics/monitor_test.go | 238 +++++++ sdk/go/client/client.gen.go | 1119 +++++++++++++++++++++++++++--- 5 files changed, 1655 insertions(+), 130 deletions(-) create mode 100644 sdk/go/analytics/monitor.go create mode 100644 sdk/go/analytics/monitor_test.go diff --git a/sdk/go/README.md b/sdk/go/README.md index 0fc288de..a21814f7 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -127,3 +127,39 @@ func signup(w http.ResponseWriter, r *http.Request) { Behind a proxy, set `ClientIP` to read the visitor's address the way your proxy reports it. Without a request, `Send` takes an `Event` with your own `VisitorKey` (a session id, never an e-mail address) instead of `IP` and `UserAgent`. + +## API monitoring + +For an analytics property of kind `api`, `Middleware` records every request your `net/http` +server answers: route pattern, method, status, duration and body sizes. It never sends a body, +header, query string or user agent. Wrap the `http.ServeMux` itself, so the pattern the mux +matched (`GET /users/{id}` becomes `/users/{id}`) is set on the request the middleware sees; a +request no pattern matched is sent with its path, and Yuva counts 404s and 405s without a pattern +as `(unmatched)`. Records go out in batches of up to 500, gzip-compressed, every 5 seconds through +the same queue and retries as events. + +```go +mux := http.NewServeMux() +mux.HandleFunc("GET /users/{id}", getUser) + +monitor, err := analytics.New(analytics.Config{ + BaseURL: "https://yuva.example.com", + APIKey: os.Getenv("YUVA_API_KEY"), + PropertyID: os.Getenv("YUVA_API_PROPERTY"), +}) +if err != nil { + return err +} +defer monitor.Close(context.Background()) + +handler := monitor.Middleware(analytics.Monitor{ + Service: "shop-api", + Consumer: func(r *http.Request) string { return apiKeyName(r) }, +})(mux) +http.ListenAndServe(":8080", handler) +``` + +`Route` names the pattern when your router is not a `ServeMux`. `SampleRate: 10` sends one in ten +successful requests (status 1 to 399), each standing for ten; errors and client errors are always +sent. `Country: true` also sends the caller's address so Yuva can count its country; it is +shortened on arrival and never stored. `SendRecord` queues a `Record` you build yourself. diff --git a/sdk/go/analytics/analytics.go b/sdk/go/analytics/analytics.go index 054245dc..ffb2fcaf 100644 --- a/sdk/go/analytics/analytics.go +++ b/sdk/go/analytics/analytics.go @@ -8,9 +8,13 @@ // visitor's browser visit of that day; none of it is stored. Requests with `DNT: 1` or // `Sec-GPC: 1` are not sent at all. // +// For an API property, Middleware records every request a net/http server answers (its route, +// method, status, duration and sizes, never a body, header or query) and sends the records in +// batches of up to 500 to `POST /v1/analytics/properties/{id}/requests` with the same key. +// // Failed batches are retried with backoff (honouring `Retry-After`); a batch Yuva refuses as a // whole (400, 401, 403, 404) is dropped and reported to Config.OnError. Call Close before the -// process exits so queued events are sent. +// process exits so queued events and records are sent. package analytics import ( @@ -20,6 +24,7 @@ import ( "errors" "fmt" "io" + "math/rand/v2" "net" "net/http" "strconv" @@ -29,11 +34,13 @@ import ( ) const ( - // MaxBatch is the most events one request carries. - MaxBatch = 100 + // MaxBatch is the most events one request carries; MaxRecordBatch the most request records. + MaxBatch = 100 + MaxRecordBatch = 500 defaultInterval = 5 * time.Second defaultQueue = 10_000 + defaultRecords = 50_000 defaultRetries = 5 defaultMinBackoff = time.Second maxBackoff = time.Minute @@ -62,6 +69,8 @@ type Config struct { FlushInterval time.Duration // QueueSize is how many events wait at most; 10,000 when zero. Events beyond it are dropped. QueueSize int + // RecordQueueSize is how many request records wait at most; 50,000 when zero. + RecordQueueSize int // MaxRetries is how often a batch is retried after a network error, 429 or 5xx; 5 when zero, // none when negative. MaxRetries int @@ -159,7 +168,8 @@ func (e *EventError) Error() string { return fmt.Sprintf("analytics: event %q refused: %s", e.Event.Name, e.Detail) } -// BatchError is a batch Yuva refused as a whole, or that failed after every retry. +// BatchError is a batch Yuva refused as a whole, or that failed after every retry; Events counts +// its events or request records. type BatchError struct { Status int Body string @@ -180,11 +190,14 @@ func (e *BatchError) Unwrap() error { return e.Err } type Client struct { cfg Config endpoint string + requests string now func() time.Time sleep func(context.Context, time.Duration) error + sample func(n int) bool mu sync.Mutex queue []Event + records []Record closed bool wake chan struct{} flushes chan chan error @@ -206,6 +219,9 @@ func New(cfg Config) (*Client, error) { if cfg.QueueSize <= 0 { cfg.QueueSize = defaultQueue } + if cfg.RecordQueueSize <= 0 { + cfg.RecordQueueSize = defaultRecords + } if cfg.MaxRetries == 0 { cfg.MaxRetries = defaultRetries } @@ -215,11 +231,14 @@ func New(cfg Config) (*Client, error) { if cfg.ClientIP == nil { cfg.ClientIP = remoteIP } + property := strings.TrimSuffix(cfg.BaseURL, "/") + "/v1/analytics/properties/" + cfg.PropertyID c := &Client{ cfg: cfg, - endpoint: strings.TrimSuffix(cfg.BaseURL, "/") + "/v1/analytics/properties/" + cfg.PropertyID + "/events", + endpoint: property + "/events", + requests: property + "/requests", now: time.Now, sleep: sleepContext, + sample: func(n int) bool { return rand.IntN(n) == 0 }, wake: make(chan struct{}, 1), flushes: make(chan chan error), stop: make(chan struct{}), @@ -385,18 +404,37 @@ func (c *Client) take() []Event { return batch } +func (c *Client) takeRecords() []Record { + c.mu.Lock() + defer c.mu.Unlock() + n := min(len(c.records), MaxRecordBatch) + batch := c.records[:n:n] + c.records = c.records[n:] + return batch +} + func (c *Client) sendAll(ctx context.Context) error { var last error for { batch := c.take() if len(batch) == 0 { - return last + break } if err := c.send(ctx, batch); err != nil { last = err c.report(err) } } + for { + batch := c.takeRecords() + if len(batch) == 0 { + return last + } + if err := c.sendRecords(ctx, batch); err != nil { + last = err + c.report(err) + } + } } func (c *Client) report(err error) { @@ -423,47 +461,58 @@ func (c *Client) send(ctx context.Context, batch []Event) error { if err != nil { return &BatchError{Events: len(batch), Err: err} } + return c.deliver(ctx, c.endpoint, body, false, len(batch), func(e int, detail string) { + c.report(&EventError{Event: batch[e], Detail: detail}) + }) +} + +// deliver posts one batch of n items, retrying network errors, 429 and 5xx, and reports the items +// an accepted batch left out. +func (c *Client) deliver(ctx context.Context, endpoint string, body []byte, gzipped bool, n int, refused func(i int, detail string)) error { backoff := c.cfg.MinBackoff for attempt := 0; ; attempt++ { - status, raw, retryAfter, err := c.post(ctx, body) + status, raw, retryAfter, err := c.post(ctx, endpoint, body, gzipped) switch { case err == nil && status == http.StatusAccepted: var res result if json.Unmarshal(raw, &res) == nil { for _, e := range res.Errors { - if e.Index >= 0 && e.Index < len(batch) { - c.report(&EventError{Event: batch[e.Index], Detail: e.Detail}) + if e.Index >= 0 && e.Index < n { + refused(e.Index, e.Detail) } } } return nil case err == nil && status != http.StatusTooManyRequests && status < 500: - return &BatchError{Status: status, Body: strings.TrimSpace(string(raw)), Events: len(batch)} + return &BatchError{Status: status, Body: strings.TrimSpace(string(raw)), Events: n} } if attempt >= c.cfg.MaxRetries || ctx.Err() != nil { if err == nil { - return &BatchError{Status: status, Body: strings.TrimSpace(string(raw)), Events: len(batch)} + return &BatchError{Status: status, Body: strings.TrimSpace(string(raw)), Events: n} } - return &BatchError{Events: len(batch), Err: err} + return &BatchError{Events: n, Err: err} } wait := backoff if retryAfter > wait { wait = retryAfter } if c.sleep(ctx, wait) != nil { - return &BatchError{Events: len(batch), Err: ctx.Err()} + return &BatchError{Events: n, Err: ctx.Err()} } backoff = min(backoff*2, maxBackoff) } } -func (c *Client) post(ctx context.Context, body []byte) (int, []byte, time.Duration, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint, bytes.NewReader(body)) +func (c *Client) post(ctx context.Context, endpoint string, body []byte, gzipped bool) (int, []byte, time.Duration, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) if err != nil { return 0, nil, 0, err } req.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) req.Header.Set("Content-Type", "application/json") + if gzipped { + req.Header.Set("Content-Encoding", "gzip") + } res, err := c.cfg.HTTPClient.Do(req) if err != nil { return 0, nil, 0, err diff --git a/sdk/go/analytics/monitor.go b/sdk/go/analytics/monitor.go new file mode 100644 index 00000000..f8906cfb --- /dev/null +++ b/sdk/go/analytics/monitor.go @@ -0,0 +1,313 @@ +package analytics + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" +) + +// Record is one request an API served, for a property of kind api. Method, Status and Duration +// are required, and Route or Path. +type Record struct { + Method string + // Route is the pattern that matched, such as /users/{id}. Without one, Path is sent and Yuva + // infers a pattern, counting 404 and 405 answers as (unmatched). + Route string + Path string + // Status is the response status, or 0 when no response was sent. + Status int + Duration time.Duration + // RequestSize and ResponseSize are body bytes; negative when unknown. + RequestSize int64 + ResponseSize int64 + // Time is when the request started; now when zero. + Time time.Time + // Consumer names who called, such as an API key's name or a client id, never a person's + // address. + Consumer string + APIVersion string + Environment string + Service string + Release string + // IP is the caller's address, sent only to find its country; Yuva shortens it on arrival and + // stores no address. + IP string + // SampleRate says the record stands for this many requests; only successful ones (status 1 to + // 399) may be sampled. + SampleRate int +} + +type wireRecord struct { + Method string `json:"method"` + Route string `json:"route,omitempty"` + Path string `json:"path,omitempty"` + Status int `json:"status"` + DurationUS int64 `json:"duration_us"` + RequestSize *int64 `json:"request_size,omitempty"` + ResponseSize *int64 `json:"response_size,omitempty"` + Time time.Time `json:"time"` + Consumer string `json:"consumer,omitempty"` + APIVersion string `json:"api_version,omitempty"` + Environment string `json:"environment,omitempty"` + Service string `json:"service,omitempty"` + Release string `json:"release,omitempty"` + IP string `json:"ip,omitempty"` + SampleRate int `json:"sample_rate,omitempty"` +} + +func (r Record) wire() wireRecord { + w := wireRecord{ + Method: r.Method, Route: r.Route, Status: r.Status, DurationUS: max(r.Duration.Microseconds(), 0), Time: r.Time.UTC(), + Consumer: r.Consumer, APIVersion: r.APIVersion, Environment: r.Environment, Service: r.Service, Release: r.Release, IP: r.IP, + } + if w.Route == "" { + w.Path = r.Path + } + if r.RequestSize >= 0 { + w.RequestSize = &r.RequestSize + } + if r.ResponseSize >= 0 { + w.ResponseSize = &r.ResponseSize + } + if r.SampleRate > 1 { + w.SampleRate = r.SampleRate + } + return w +} + +// RecordError is a request record Yuva left out of an accepted batch as invalid. +type RecordError struct { + Record Record + Detail string +} + +func (e *RecordError) Error() string { + route := e.Record.Route + if route == "" { + route = e.Record.Path + } + return fmt.Sprintf("analytics: request record %s %s refused: %s", e.Record.Method, route, e.Detail) +} + +// ErrRecordQueueFull is reported when a request record is dropped because the queue holds +// Config.RecordQueueSize records. +var ErrRecordQueueFull = errors.New("analytics: the request record queue is full; record dropped") + +// SendRecord queues a request record as it is. +func (c *Client) SendRecord(r Record) error { + if r.Method == "" || (r.Route == "" && r.Path == "") { + return errors.New("analytics: a request record needs a Method and a Route or Path") + } + if r.Time.IsZero() { + r.Time = c.now() + } + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return ErrClosed + } + if len(c.records) >= c.cfg.RecordQueueSize { + c.mu.Unlock() + c.report(ErrRecordQueueFull) + return ErrRecordQueueFull + } + c.records = append(c.records, r) + full := len(c.records) >= MaxRecordBatch + c.mu.Unlock() + if full { + select { + case c.wake <- struct{}{}: + default: + } + } + return nil +} + +func (c *Client) sendRecords(ctx context.Context, batch []Record) error { + wire := make([]wireRecord, len(batch)) + for i, r := range batch { + wire[i] = r.wire() + } + raw, err := json.Marshal(map[string]any{"records": wire}) + if err != nil { + return &BatchError{Events: len(batch), Err: err} + } + var body bytes.Buffer + zw := gzip.NewWriter(&body) + if _, err := zw.Write(raw); err != nil { + return &BatchError{Events: len(batch), Err: err} + } + if err := zw.Close(); err != nil { + return &BatchError{Events: len(batch), Err: err} + } + return c.deliver(ctx, c.requests, body.Bytes(), true, len(batch), func(i int, detail string) { + c.report(&RecordError{Record: batch[i], Detail: detail}) + }) +} + +// Monitor configures Middleware. Every field is optional. +type Monitor struct { + // Route names a request's route pattern. When nil, Middleware reads r.Pattern after the + // handler ran (http.ServeMux sets it), without its method and host; a request no pattern + // matched is sent with its path. + Route func(r *http.Request) string + // Consumer names who called, such as the API key's name or a client id; never a person's + // address. + Consumer func(r *http.Request) string + APIVersion func(r *http.Request) string + // Environment is production when empty. + Environment string + Service string + Release string + // SampleRate sends one in SampleRate successful requests (status 1 to 399), each standing for + // SampleRate requests; errors and client errors are always sent. Every request when 0 or 1. + SampleRate int + // Country sends the caller's address (Config.ClientIP) so Yuva can count the request's + // country; the address is shortened on arrival and never stored. + Country bool +} + +// Middleware records every request the handler answers. Wrap the http.ServeMux itself, so the +// pattern it matched is set on the request Middleware sees: +// +// http.ListenAndServe(":8080", client.Middleware(analytics.Monitor{Service: "shop"})(mux)) +// +// A handler that panics is recorded with the status it wrote, or 0 when it wrote none, and the +// panic goes on. +func (c *Client) Middleware(m Monitor) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := c.now() + rw := &responseWriter{ResponseWriter: w} + var body *countingBody + if r.Body != nil && r.Body != http.NoBody { + body = &countingBody{ReadCloser: r.Body} + r.Body = body + } + defer func() { + v := recover() + status := rw.status + if status == 0 && v == nil { + status = http.StatusOK + } + c.recordRequest(m, r, status, c.now().Sub(start), body, rw.written, start) + if v != nil { + panic(v) + } + }() + next.ServeHTTP(rw, r) + }) + } +} + +func (c *Client) recordRequest(m Monitor, r *http.Request, status int, took time.Duration, body *countingBody, written int64, start time.Time) { + weight := 1 + if m.SampleRate > 1 && status > 0 && status < 400 { + if !c.sample(m.SampleRate) { + return + } + weight = m.SampleRate + } + rec := Record{Method: r.Method, Status: status, Duration: took, RequestSize: r.ContentLength, ResponseSize: written, Time: start, + Environment: m.Environment, Service: m.Service, Release: m.Release, SampleRate: weight} + if body != nil && body.n > 0 { + rec.RequestSize = body.n + } + if r.ContentLength < 0 && (body == nil || body.n == 0) { + rec.RequestSize = -1 + } + if m.Route != nil { + rec.Route = m.Route(r) + } else { + rec.Route = patternRoute(r.Pattern) + } + if rec.Route == "" { + rec.Path = r.URL.Path + } + if m.Consumer != nil { + rec.Consumer = m.Consumer(r) + } + if m.APIVersion != nil { + rec.APIVersion = m.APIVersion(r) + } + if m.Country { + rec.IP = c.cfg.ClientIP(r) + } + _ = c.SendRecord(rec) +} + +// patternRoute is a ServeMux pattern without its method and host: "GET example.com/users/{id}" +// is /users/{id}. +func patternRoute(p string) string { + if _, rest, ok := strings.Cut(p, " "); ok { + p = strings.TrimSpace(rest) + } + if i := strings.IndexByte(p, '/'); i > 0 { + p = p[i:] + } + if !strings.HasPrefix(p, "/") { + return "" + } + return p +} + +type countingBody struct { + io.ReadCloser + n int64 +} + +func (b *countingBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + b.n += int64(n) + return n, err +} + +// responseWriter notes the status and the body bytes; Unwrap lets http.ResponseController reach +// the connection's own writer. +type responseWriter struct { + http.ResponseWriter + status int + written int64 +} + +func (w *responseWriter) WriteHeader(status int) { + if w.status == 0 && status >= 200 { + w.status = status + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *responseWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + n, err := w.ResponseWriter.Write(p) + w.written += int64(n) + return n, err +} + +func (w *responseWriter) Flush() { + if w.status == 0 { + w.status = http.StatusOK + } + _ = http.NewResponseController(w.ResponseWriter).Flush() +} + +func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + conn, rw, err := http.NewResponseController(w.ResponseWriter).Hijack() + if err == nil && w.status == 0 { + w.status = http.StatusSwitchingProtocols + } + return conn, rw, err +} + +func (w *responseWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } diff --git a/sdk/go/analytics/monitor_test.go b/sdk/go/analytics/monitor_test.go new file mode 100644 index 00000000..9dc3bdf8 --- /dev/null +++ b/sdk/go/analytics/monitor_test.go @@ -0,0 +1,238 @@ +package analytics + +import ( + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +type recordSink struct { + mu sync.Mutex + batches [][]map[string]any + answer func(w http.ResponseWriter) +} + +func (s *recordSink) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/analytics/properties/prop-1/requests" || r.Header.Get("Content-Encoding") != "gzip" || r.Header.Get("Authorization") != "Bearer yuva_key" { + w.WriteHeader(http.StatusNotFound) + return + } + zr, err := gzip.NewReader(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + var body struct { + Records []map[string]any `json:"records"` + } + _ = json.NewDecoder(zr).Decode(&body) + s.mu.Lock() + s.batches = append(s.batches, body.Records) + answer := s.answer + s.mu.Unlock() + if answer != nil { + answer(w) + return + } + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"accepted": 1, "errors": []}`)) +} + +func (s *recordSink) all() []map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + var out []map[string]any + for _, b := range s.batches { + out = append(out, b...) + } + return out +} + +func newMonitorClient(t *testing.T, sink *recordSink) (*Client, *[]error) { + t.Helper() + srv := httptest.NewServer(sink) + t.Cleanup(srv.Close) + var mu sync.Mutex + errs := &[]error{} + c, err := New(Config{BaseURL: srv.URL, APIKey: "yuva_key", PropertyID: "prop-1", FlushInterval: time.Hour, OnError: func(err error) { + mu.Lock() + defer mu.Unlock() + *errs = append(*errs, err) + }}) + if err != nil { + t.Fatal(err) + } + c.sleep = func(ctx context.Context, d time.Duration) error { return ctx.Err() } + t.Cleanup(func() { _ = c.Close(context.Background()) }) + return c, errs +} + +func serve(h http.Handler, method, target, body string) *httptest.ResponseRecorder { + var rd io.Reader + if body != "" { + rd = strings.NewReader(body) + } + r := httptest.NewRequest(method, target, rd) + r.RemoteAddr = "203.0.113.7:4000" + w := httptest.NewRecorder() + h.ServeHTTP(w, r) + return w +} + +func TestMiddlewareRecordsTheMuxPattern(t *testing.T) { + sink := &recordSink{} + c, errs := newMonitorClient(t, sink) + mux := http.NewServeMux() + mux.HandleFunc("GET /users/{id}", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"id":"` + r.PathValue("id") + `"}`)) + }) + mux.HandleFunc("POST api.example.com/orders", func(w http.ResponseWriter, r *http.Request) { + _, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusCreated) + }) + mux.HandleFunc("/boom", func(http.ResponseWriter, *http.Request) { panic("boom") }) + h := c.Middleware(Monitor{Service: "shop", Environment: "staging", Country: true, + Consumer: func(r *http.Request) string { return r.Header.Get("X-Client") }})(mux) + + serve(h, http.MethodGet, "http://api.example.com/users/42?token=secret", "") + serve(h, http.MethodPost, "http://api.example.com/orders", `{"sku":"A-1"}`) + serve(h, http.MethodGet, "http://api.example.com/wp-login.php?x=1", "") + serve(h, http.MethodDelete, "http://api.example.com/users/42", "") + func() { + defer func() { + if recover() == nil { + t.Error("the panic did not go on") + } + }() + serve(h, http.MethodGet, "http://api.example.com/boom", "") + }() + if err := c.Flush(context.Background()); err != nil { + t.Fatal(err) + } + recs := sink.all() + if len(recs) != 5 || len(*errs) != 0 { + t.Fatalf("%d records, errors %v", len(recs), *errs) + } + want := []map[string]any{ + {"method": "GET", "route": "/users/{id}", "status": 200.0, "response_size": 11.0, "service": "shop", "environment": "staging", "ip": "203.0.113.7"}, + {"method": "POST", "route": "/orders", "status": 201.0, "request_size": 13.0, "response_size": 0.0}, + {"method": "GET", "path": "/wp-login.php", "status": 404.0}, + {"method": "DELETE", "path": "/users/42", "status": 405.0}, + {"method": "GET", "route": "/boom", "status": 0.0}, + } + for i, w := range want { + for k, v := range w { + if recs[i][k] != v { + t.Fatalf("record %d: %s = %v, want %v (%v)", i, k, recs[i][k], v, recs[i]) + } + } + if _, ok := recs[i]["duration_us"].(float64); !ok || recs[i]["time"] == nil { + t.Fatalf("record %d has no duration or time: %v", i, recs[i]) + } + for _, k := range []string{"query", "headers", "user_agent", "body"} { + if _, ok := recs[i][k]; ok { + t.Fatalf("record %d carries %s", i, k) + } + } + } + if strings.Contains(recs[0]["route"].(string)+recs[2]["path"].(string), "?") { + t.Fatal("a query string was sent") + } +} + +func TestMiddlewareRouteBatchesAndSampling(t *testing.T) { + sink := &recordSink{} + c, _ := newMonitorClient(t, sink) + n := 0 + c.sample = func(rate int) bool { + n++ + return n%rate == 0 + } + h := c.Middleware(Monitor{SampleRate: 4, Route: func(*http.Request) string { return "/things/{id}" }})(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("fail") != "" { + w.WriteHeader(http.StatusBadGateway) + } + })) + for range 2000 { + serve(h, http.MethodGet, "http://api.example.com/things/1", "") + } + deadline := time.Now().Add(5 * time.Second) + for len(sink.all()) < MaxRecordBatch && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if got := len(sink.all()); got != MaxRecordBatch { + t.Fatalf("%d records before the interval, want a full batch of %d", got, MaxRecordBatch) + } + for range 3 { + serve(h, http.MethodGet, "http://api.example.com/things/1?fail=1", "") + } + if err := c.Flush(context.Background()); err != nil { + t.Fatal(err) + } + recs := sink.all() + failed := 0 + for _, r := range recs { + if r["route"] != "/things/{id}" { + t.Fatalf("route %v", r) + } + switch r["status"] { + case 200.0: + if r["sample_rate"] != 4.0 { + t.Fatalf("a sampled record %v", r) + } + case 502.0: + failed++ + if r["sample_rate"] != nil { + t.Fatalf("an error was sampled: %v", r) + } + } + } + if len(recs) != 503 || failed != 3 { + t.Fatalf("%d records, %d errors", len(recs), failed) + } +} + +func TestRecordErrorsAndQueue(t *testing.T) { + sink := &recordSink{answer: func(w http.ResponseWriter) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"accepted": 1, "errors": [{"index": 1, "detail": "status must be 100 to 599, or 0 when no response was sent"}]}`)) + }} + c, errs := newMonitorClient(t, sink) + _ = c.SendRecord(Record{Method: "GET", Route: "/a", Status: 200, Duration: time.Millisecond}) + _ = c.SendRecord(Record{Method: "GET", Route: "/b", Status: 900, Duration: time.Millisecond}) + if err := c.SendRecord(Record{Method: "GET", Status: 200}); err == nil { + t.Fatal("a record without a route or path was queued") + } + if err := c.Flush(context.Background()); err != nil { + t.Fatal(err) + } + var re *RecordError + if len(*errs) != 1 || !errors.As((*errs)[0], &re) || re.Record.Route != "/b" { + t.Fatalf("record errors %v", *errs) + } + if got := sink.all()[0]["duration_us"]; got != 1000.0 { + t.Fatalf("duration %v", got) + } +} + +func TestPatternRoute(t *testing.T) { + for in, want := range map[string]string{ + "GET /users/{id}": "/users/{id}", + "POST api.example.com/orders/{id}": "/orders/{id}", + "example.com/": "/", + "/static/": "/static/", + "": "", + } { + if got := patternRoute(in); got != want { + t.Errorf("patternRoute(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/sdk/go/client/client.gen.go b/sdk/go/client/client.gen.go index 2495f5dc..57c1b4f9 100644 --- a/sdk/go/client/client.gen.go +++ b/sdk/go/client/client.gen.go @@ -221,6 +221,7 @@ func (e AnalyticsConsentFeature) Valid() bool { // Defines values for AnalyticsDataset. const ( + AnalyticsDatasetApi AnalyticsDataset = "api" AnalyticsDatasetSupport AnalyticsDataset = "support" AnalyticsDatasetTraffic AnalyticsDataset = "traffic" ) @@ -228,6 +229,8 @@ const ( // Valid indicates whether the value is a known member of the AnalyticsDataset enum. func (e AnalyticsDataset) Valid() bool { switch e { + case AnalyticsDatasetApi: + return true case AnalyticsDatasetSupport: return true case AnalyticsDatasetTraffic: @@ -248,11 +251,14 @@ const ( AnalyticsByDay AnalyticsDimension = "time:day" AnalyticsByDevice AnalyticsDimension = "device" AnalyticsByEntryPage AnalyticsDimension = "entry_page" + AnalyticsByEnvironment AnalyticsDimension = "environment" AnalyticsByEvent AnalyticsDimension = "event" AnalyticsByExitPage AnalyticsDimension = "exit_page" AnalyticsByHostname AnalyticsDimension = "hostname" AnalyticsByHour AnalyticsDimension = "time:hour" AnalyticsByLanguage AnalyticsDimension = "language" + AnalyticsByMethod AnalyticsDimension = "method" + AnalyticsByMinute AnalyticsDimension = "time:minute" AnalyticsByMonth AnalyticsDimension = "time:month" AnalyticsByOs AnalyticsDimension = "os" AnalyticsByOsVersion AnalyticsDimension = "os_version" @@ -261,8 +267,11 @@ const ( AnalyticsByProp AnalyticsDimension = "prop" AnalyticsByReferrer AnalyticsDimension = "referrer" AnalyticsByRegion AnalyticsDimension = "region" + AnalyticsByRoute AnalyticsDimension = "route" AnalyticsByScreenSize AnalyticsDimension = "screen_size" AnalyticsBySource AnalyticsDimension = "source" + AnalyticsByStatus AnalyticsDimension = "status" + AnalyticsByStatusClass AnalyticsDimension = "status_class" AnalyticsByUtmCampaign AnalyticsDimension = "utm_campaign" AnalyticsByUtmContent AnalyticsDimension = "utm_content" AnalyticsByUtmMedium AnalyticsDimension = "utm_medium" @@ -293,6 +302,8 @@ func (e AnalyticsDimension) Valid() bool { return true case AnalyticsByEntryPage: return true + case AnalyticsByEnvironment: + return true case AnalyticsByEvent: return true case AnalyticsByExitPage: @@ -303,6 +314,10 @@ func (e AnalyticsDimension) Valid() bool { return true case AnalyticsByLanguage: return true + case AnalyticsByMethod: + return true + case AnalyticsByMinute: + return true case AnalyticsByMonth: return true case AnalyticsByOs: @@ -319,10 +334,16 @@ func (e AnalyticsDimension) Valid() bool { return true case AnalyticsByRegion: return true + case AnalyticsByRoute: + return true case AnalyticsByScreenSize: return true case AnalyticsBySource: return true + case AnalyticsByStatus: + return true + case AnalyticsByStatusClass: + return true case AnalyticsByUtmCampaign: return true case AnalyticsByUtmContent: @@ -442,10 +463,12 @@ const ( AnalyticsDimCountry AnalyticsFilterDimension = "country" AnalyticsDimDevice AnalyticsFilterDimension = "device" AnalyticsDimEntryPage AnalyticsFilterDimension = "entry_page" + AnalyticsDimEnvironment AnalyticsFilterDimension = "environment" AnalyticsDimEvent AnalyticsFilterDimension = "event" AnalyticsDimExitPage AnalyticsFilterDimension = "exit_page" AnalyticsDimHostname AnalyticsFilterDimension = "hostname" AnalyticsDimLanguage AnalyticsFilterDimension = "language" + AnalyticsDimMethod AnalyticsFilterDimension = "method" AnalyticsDimOs AnalyticsFilterDimension = "os" AnalyticsDimOsVersion AnalyticsFilterDimension = "os_version" AnalyticsDimPage AnalyticsFilterDimension = "page" @@ -453,8 +476,11 @@ const ( AnalyticsDimProp AnalyticsFilterDimension = "prop" AnalyticsDimReferrer AnalyticsFilterDimension = "referrer" AnalyticsDimRegion AnalyticsFilterDimension = "region" + AnalyticsDimRoute AnalyticsFilterDimension = "route" AnalyticsDimScreenSize AnalyticsFilterDimension = "screen_size" AnalyticsDimSource AnalyticsFilterDimension = "source" + AnalyticsDimStatus AnalyticsFilterDimension = "status" + AnalyticsDimStatusClass AnalyticsFilterDimension = "status_class" AnalyticsDimUtmCampaign AnalyticsFilterDimension = "utm_campaign" AnalyticsDimUtmContent AnalyticsFilterDimension = "utm_content" AnalyticsDimUtmMedium AnalyticsFilterDimension = "utm_medium" @@ -482,6 +508,8 @@ func (e AnalyticsFilterDimension) Valid() bool { return true case AnalyticsDimEntryPage: return true + case AnalyticsDimEnvironment: + return true case AnalyticsDimEvent: return true case AnalyticsDimExitPage: @@ -490,6 +518,8 @@ func (e AnalyticsFilterDimension) Valid() bool { return true case AnalyticsDimLanguage: return true + case AnalyticsDimMethod: + return true case AnalyticsDimOs: return true case AnalyticsDimOsVersion: @@ -504,10 +534,16 @@ func (e AnalyticsFilterDimension) Valid() bool { return true case AnalyticsDimRegion: return true + case AnalyticsDimRoute: + return true case AnalyticsDimScreenSize: return true case AnalyticsDimSource: return true + case AnalyticsDimStatus: + return true + case AnalyticsDimStatusClass: + return true case AnalyticsDimUtmCampaign: return true case AnalyticsDimUtmContent: @@ -596,33 +632,49 @@ func (e AnalyticsInsightKind) Valid() bool { // Defines values for AnalyticsMetric. const ( - AnalyticsMetricBounceRate AnalyticsMetric = "bounce_rate" - AnalyticsMetricCompletions AnalyticsMetric = "completions" - AnalyticsMetricConversations AnalyticsMetric = "conversations" - AnalyticsMetricConversionRate AnalyticsMetric = "conversion_rate" - AnalyticsMetricConversions AnalyticsMetric = "conversions" - AnalyticsMetricDiscussions AnalyticsMetric = "discussions" - AnalyticsMetricEvents AnalyticsMetric = "events" - AnalyticsMetricFeedback AnalyticsMetric = "feedback" - AnalyticsMetricHelpfulRate AnalyticsMetric = "helpful_rate" - AnalyticsMetricPageviews AnalyticsMetric = "pageviews" - AnalyticsMetricRatingsDown AnalyticsMetric = "ratings_down" - AnalyticsMetricRatingsUp AnalyticsMetric = "ratings_up" - AnalyticsMetricRevenue AnalyticsMetric = "revenue" - AnalyticsMetricScrollDepth AnalyticsMetric = "scroll_depth" - AnalyticsMetricSupportRate AnalyticsMetric = "support_rate" - AnalyticsMetricTimeOnPage AnalyticsMetric = "time_on_page" - AnalyticsMetricViewsPerVisit AnalyticsMetric = "views_per_visit" - AnalyticsMetricVisitDuration AnalyticsMetric = "visit_duration" - AnalyticsMetricVisitors AnalyticsMetric = "visitors" - AnalyticsMetricVisits AnalyticsMetric = "visits" + AnalyticsMetricAvgDuration AnalyticsMetric = "avg_duration" + AnalyticsMetricBounceRate AnalyticsMetric = "bounce_rate" + AnalyticsMetricClientErrorRate AnalyticsMetric = "client_error_rate" + AnalyticsMetricClientErrors AnalyticsMetric = "client_errors" + AnalyticsMetricCompletions AnalyticsMetric = "completions" + AnalyticsMetricConversations AnalyticsMetric = "conversations" + AnalyticsMetricConversionRate AnalyticsMetric = "conversion_rate" + AnalyticsMetricConversions AnalyticsMetric = "conversions" + AnalyticsMetricDiscussions AnalyticsMetric = "discussions" + AnalyticsMetricErrorRate AnalyticsMetric = "error_rate" + AnalyticsMetricErrors AnalyticsMetric = "errors" + AnalyticsMetricEvents AnalyticsMetric = "events" + AnalyticsMetricFeedback AnalyticsMetric = "feedback" + AnalyticsMetricHelpfulRate AnalyticsMetric = "helpful_rate" + AnalyticsMetricP50 AnalyticsMetric = "p50" + AnalyticsMetricP95 AnalyticsMetric = "p95" + AnalyticsMetricP99 AnalyticsMetric = "p99" + AnalyticsMetricPageviews AnalyticsMetric = "pageviews" + AnalyticsMetricRatingsDown AnalyticsMetric = "ratings_down" + AnalyticsMetricRatingsUp AnalyticsMetric = "ratings_up" + AnalyticsMetricRequests AnalyticsMetric = "requests" + AnalyticsMetricRequestsPerMinute AnalyticsMetric = "requests_per_minute" + AnalyticsMetricRevenue AnalyticsMetric = "revenue" + AnalyticsMetricScrollDepth AnalyticsMetric = "scroll_depth" + AnalyticsMetricSupportRate AnalyticsMetric = "support_rate" + AnalyticsMetricTimeOnPage AnalyticsMetric = "time_on_page" + AnalyticsMetricViewsPerVisit AnalyticsMetric = "views_per_visit" + AnalyticsMetricVisitDuration AnalyticsMetric = "visit_duration" + AnalyticsMetricVisitors AnalyticsMetric = "visitors" + AnalyticsMetricVisits AnalyticsMetric = "visits" ) // Valid indicates whether the value is a known member of the AnalyticsMetric enum. func (e AnalyticsMetric) Valid() bool { switch e { + case AnalyticsMetricAvgDuration: + return true case AnalyticsMetricBounceRate: return true + case AnalyticsMetricClientErrorRate: + return true + case AnalyticsMetricClientErrors: + return true case AnalyticsMetricCompletions: return true case AnalyticsMetricConversations: @@ -633,18 +685,32 @@ func (e AnalyticsMetric) Valid() bool { return true case AnalyticsMetricDiscussions: return true + case AnalyticsMetricErrorRate: + return true + case AnalyticsMetricErrors: + return true case AnalyticsMetricEvents: return true case AnalyticsMetricFeedback: return true case AnalyticsMetricHelpfulRate: return true + case AnalyticsMetricP50: + return true + case AnalyticsMetricP95: + return true + case AnalyticsMetricP99: + return true case AnalyticsMetricPageviews: return true case AnalyticsMetricRatingsDown: return true case AnalyticsMetricRatingsUp: return true + case AnalyticsMetricRequests: + return true + case AnalyticsMetricRequestsPerMinute: + return true case AnalyticsMetricRevenue: return true case AnalyticsMetricScrollDepth: @@ -705,19 +771,41 @@ func (e AnalyticsPlatform) Valid() bool { } } +// Defines values for AnalyticsPropertyKind. +const ( + AnalyticsKindApi AnalyticsPropertyKind = "api" + AnalyticsKindSite AnalyticsPropertyKind = "site" +) + +// Valid indicates whether the value is a known member of the AnalyticsPropertyKind enum. +func (e AnalyticsPropertyKind) Valid() bool { + switch e { + case AnalyticsKindApi: + return true + case AnalyticsKindSite: + return true + default: + return false + } +} + // Defines values for AnalyticsRangePeriod. const ( - AnalyticsLast12Months AnalyticsRangePeriod = "12mo" - AnalyticsLast28Days AnalyticsRangePeriod = "28d" - AnalyticsLast30Days AnalyticsRangePeriod = "30d" - AnalyticsLast6Months AnalyticsRangePeriod = "6mo" - AnalyticsLast7Days AnalyticsRangePeriod = "7d" - AnalyticsLast90Days AnalyticsRangePeriod = "90d" - AnalyticsLastMonth AnalyticsRangePeriod = "last_month" - AnalyticsThisMonth AnalyticsRangePeriod = "month" - AnalyticsThisYear AnalyticsRangePeriod = "year" - AnalyticsToday AnalyticsRangePeriod = "today" - AnalyticsYesterday AnalyticsRangePeriod = "yesterday" + AnalyticsLast12Months AnalyticsRangePeriod = "12mo" + AnalyticsLast15Minutes AnalyticsRangePeriod = "15m" + AnalyticsLast24Hours AnalyticsRangePeriod = "24h" + AnalyticsLast28Days AnalyticsRangePeriod = "28d" + AnalyticsLast30Days AnalyticsRangePeriod = "30d" + AnalyticsLast6Hours AnalyticsRangePeriod = "6h" + AnalyticsLast6Months AnalyticsRangePeriod = "6mo" + AnalyticsLast7Days AnalyticsRangePeriod = "7d" + AnalyticsLast90Days AnalyticsRangePeriod = "90d" + AnalyticsLastHour AnalyticsRangePeriod = "1h" + AnalyticsLastMonth AnalyticsRangePeriod = "last_month" + AnalyticsThisMonth AnalyticsRangePeriod = "month" + AnalyticsThisYear AnalyticsRangePeriod = "year" + AnalyticsToday AnalyticsRangePeriod = "today" + AnalyticsYesterday AnalyticsRangePeriod = "yesterday" ) // Valid indicates whether the value is a known member of the AnalyticsRangePeriod enum. @@ -725,16 +813,24 @@ func (e AnalyticsRangePeriod) Valid() bool { switch e { case AnalyticsLast12Months: return true + case AnalyticsLast15Minutes: + return true + case AnalyticsLast24Hours: + return true case AnalyticsLast28Days: return true case AnalyticsLast30Days: return true + case AnalyticsLast6Hours: + return true case AnalyticsLast6Months: return true case AnalyticsLast7Days: return true case AnalyticsLast90Days: return true + case AnalyticsLastHour: + return true case AnalyticsLastMonth: return true case AnalyticsThisMonth: @@ -795,6 +891,39 @@ func (e AnalyticsReportFrequency) Valid() bool { } } +// Defines values for AnalyticsResultMetaCapsReached. +const ( + AnalyticsCapApiVersions AnalyticsResultMetaCapsReached = "api_versions" + AnalyticsCapAppVersions AnalyticsResultMetaCapsReached = "app_versions" + AnalyticsCapConsumers AnalyticsResultMetaCapsReached = "consumers" + AnalyticsCapDimensionRows AnalyticsResultMetaCapsReached = "dimension_rows" + AnalyticsCapEnvironments AnalyticsResultMetaCapsReached = "environments" + AnalyticsCapRoutes AnalyticsResultMetaCapsReached = "routes" + AnalyticsCapServices AnalyticsResultMetaCapsReached = "services" +) + +// Valid indicates whether the value is a known member of the AnalyticsResultMetaCapsReached enum. +func (e AnalyticsResultMetaCapsReached) Valid() bool { + switch e { + case AnalyticsCapApiVersions: + return true + case AnalyticsCapAppVersions: + return true + case AnalyticsCapConsumers: + return true + case AnalyticsCapDimensionRows: + return true + case AnalyticsCapEnvironments: + return true + case AnalyticsCapRoutes: + return true + case AnalyticsCapServices: + return true + default: + return false + } +} + // Defines values for AnalyticsResultMetaServedFrom. const ( AnalyticsServedMixed AnalyticsResultMetaServedFrom = "mixed" @@ -4500,6 +4629,24 @@ func (e QueryAnalyticsParamsFormat) Valid() bool { } } +// Defines values for SendAnalyticsRequestsParamsContentEncoding. +const ( + SendAnalyticsRequestsParamsContentEncodingGzip SendAnalyticsRequestsParamsContentEncoding = "gzip" + SendAnalyticsRequestsParamsContentEncodingIdentity SendAnalyticsRequestsParamsContentEncoding = "identity" +) + +// Valid indicates whether the value is a known member of the SendAnalyticsRequestsParamsContentEncoding enum. +func (e SendAnalyticsRequestsParamsContentEncoding) Valid() bool { + switch e { + case SendAnalyticsRequestsParamsContentEncodingGzip: + return true + case SendAnalyticsRequestsParamsContentEncodingIdentity: + return true + default: + return false + } +} + // Defines values for GetAnalyticsRetentionParamsGrain. const ( AnalyticsRetentionDay GetAnalyticsRetentionParamsGrain = "day" @@ -4935,6 +5082,14 @@ type AnalyticsDataDeletion struct { // `support_rate` with pages of fewer than 100 page views after the others unless `order_by` // says otherwise. Days of the past come from the support rollup, today and days not yet rolled // up from the inbox itself. +// +// `api` is an API property's requests (`sendAnalyticsRequests`), and the only dataset an API +// property takes (`422 property_kind` otherwise, and for `api` on a site). It takes the API +// metrics of `AnalyticsMetric`, the API dimensions of `AnalyticsDimension` and filters on +// them, no goal, prop, segment, comparison or insights. Closed local days come from daily +// rows, closed hours from hourly rows and the rest from minute rows, which processes write +// every 10 seconds; `meta.served_from` is `rollups` when only daily and hourly rows answered. +// Rows are ordered by `requests` unless `order_by` says otherwise. type AnalyticsDataset string // AnalyticsDimension A filter dimension (`AnalyticsFilterDimension`), or a time grain: `time:hour` (local hours @@ -4944,6 +5099,11 @@ type AnalyticsDataset string // range is present. `event` breaks custom and built-in events down by name, `prop` by the // value of the event prop the query's `prop` names; either takes at most one other dimension, // a time grain other than `time:hour` or a visit dimension. +// +// With `dataset: api`, two of `route`, `method`, `status`, `status_class`, `environment` and +// one grain: `time:minute` (minutes, `2026-10-12T14:05` in the property's time zone, inside +// the last `YUVA_MONITOR_MINUTE_DAYS` days and at most 1,440 of them), `time:hour` (local +// hours) or `time:day`. type AnalyticsDimension string // AnalyticsDomain A host name whose pages may send events, without scheme or port: `example.com`, @@ -5185,6 +5345,12 @@ type AnalyticsExportCreate struct { // for events without it: as // a filter of visits they keep the visits with such an event; in an `event` or `prop` // breakdown they keep the matching events. + // + // With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for + // requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 + // routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response + // was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's + // environment (`production` by default, `(other)` past 20). Dimension *AnalyticsFilterDimension `json:"dimension,omitempty"` Format *AnalyticsExportFormat `json:"format,omitempty"` Kind AnalyticsExportKind `json:"kind"` @@ -5193,7 +5359,8 @@ type AnalyticsExportCreate struct { // Range Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, // `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and // `year` are the current calendar month and year to date; `last_month` the one before), or - // `from` and `to` (inclusive, at most 400 days apart). + // `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also + // be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. Range *AnalyticsRange `json:"range,omitempty"` } @@ -5229,6 +5396,12 @@ type AnalyticsFilter struct { // for events without it: as // a filter of visits they keep the visits with such an event; in an `event` or `prop` // breakdown they keep the matching events. + // + // With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for + // requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 + // routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response + // was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's + // environment (`production` by default, `(other)` past 20). Dimension AnalyticsFilterDimension `json:"dimension"` Op AnalyticsFilterOp `json:"op"` @@ -5256,6 +5429,12 @@ type AnalyticsFilterOp string // for events without it: as // a filter of visits they keep the visits with such an event; in an `event` or `prop` // breakdown they keep the matching events. +// +// With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for +// requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 +// routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response +// was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's +// environment (`production` by default, `(other)` past 20). type AnalyticsFilterDimension string // AnalyticsFunnel defines model for AnalyticsFunnel. @@ -5489,6 +5668,13 @@ type AnalyticsInsight struct { // `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), // `discussions` (discussion threads started on it) and `support_rate` (conversations per // 1,000 page views), next to `visitors` and `pageviews`. + // + // With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests + // per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no + // response was sent), `error_rate` and `client_error_rate` (percent of requests), + // `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds + // that this share of requests stayed within, from sketches accurate to 1.1%) and + // `avg_duration` (milliseconds). Metric AnalyticsMetric `json:"metric"` // Previous The comparison range's number (0 for `metric_new` and `new_value`), or the range's overall `support_rate` for a hotspot. @@ -5521,22 +5707,39 @@ type AnalyticsInsightKind string // `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), // `discussions` (discussion threads started on it) and `support_rate` (conversations per // 1,000 page views), next to `visitors` and `pageviews`. +// +// With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests +// per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no +// response was sent), `error_rate` and `client_error_rate` (percent of requests), +// `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds +// that this share of requests stayed within, from sketches accurate to 1.1%) and +// `avg_duration` (milliseconds). type AnalyticsMetric string // AnalyticsMetrics The metrics the query asked for. type AnalyticsMetrics struct { - BounceRate *float64 `json:"bounce_rate,omitempty"` - Completions *float64 `json:"completions,omitempty"` - Conversations *float64 `json:"conversations,omitempty"` - ConversionRate *float64 `json:"conversion_rate,omitempty"` - Conversions *float64 `json:"conversions,omitempty"` - Discussions *float64 `json:"discussions,omitempty"` - Events *float64 `json:"events,omitempty"` - Feedback *float64 `json:"feedback,omitempty"` - HelpfulRate *float64 `json:"helpful_rate,omitempty"` - Pageviews *float64 `json:"pageviews,omitempty"` - RatingsDown *float64 `json:"ratings_down,omitempty"` - RatingsUp *float64 `json:"ratings_up,omitempty"` + AvgDuration *float64 `json:"avg_duration,omitempty"` + BounceRate *float64 `json:"bounce_rate,omitempty"` + ClientErrorRate *float64 `json:"client_error_rate,omitempty"` + ClientErrors *float64 `json:"client_errors,omitempty"` + Completions *float64 `json:"completions,omitempty"` + Conversations *float64 `json:"conversations,omitempty"` + ConversionRate *float64 `json:"conversion_rate,omitempty"` + Conversions *float64 `json:"conversions,omitempty"` + Discussions *float64 `json:"discussions,omitempty"` + ErrorRate *float64 `json:"error_rate,omitempty"` + Errors *float64 `json:"errors,omitempty"` + Events *float64 `json:"events,omitempty"` + Feedback *float64 `json:"feedback,omitempty"` + HelpfulRate *float64 `json:"helpful_rate,omitempty"` + P50 *float64 `json:"p50,omitempty"` + P95 *float64 `json:"p95,omitempty"` + P99 *float64 `json:"p99,omitempty"` + Pageviews *float64 `json:"pageviews,omitempty"` + RatingsDown *float64 `json:"ratings_down,omitempty"` + RatingsUp *float64 `json:"ratings_up,omitempty"` + Requests *float64 `json:"requests,omitempty"` + RequestsPerMinute *float64 `json:"requests_per_minute,omitempty"` // Revenue Per currency, most first. Revenue *[]AnalyticsRevenue `json:"revenue,omitempty"` @@ -5580,6 +5783,13 @@ type AnalyticsOrder struct { // `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), // `discussions` (discussion threads started on it) and `support_rate` (conversations per // 1,000 page views), next to `visitors` and `pageviews`. + // + // With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests + // per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no + // response was sent), `error_rate` and `client_error_rate` (percent of requests), + // `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds + // that this share of requests stayed within, from sketches accurate to 1.1%) and + // `avg_duration` (milliseconds). Metric AnalyticsMetric `json:"metric"` } @@ -5606,8 +5816,11 @@ type AnalyticsPathRule struct { // apps send without an `Origin`. Server-side events (`sendAnalyticsEvents`) need no platform. type AnalyticsPlatform string -// AnalyticsProperty A website or app the workspace measures. +// AnalyticsProperty A website, app or API the workspace measures. type AnalyticsProperty struct { + // ApdexMs An API's Apdex threshold T in milliseconds; a request is satisfied within T and tolerated within 4T. + ApdexMs int32 `json:"apdex_ms"` + // Audience `children` keeps location at country level and allows no consent features or tracked props. Audience AnalyticsAudience `json:"audience"` @@ -5631,11 +5844,21 @@ type AnalyticsProperty struct { // InboxId The linked inbox, whose agents can read the property; absent when none. InboxId *uuid.UUID `json:"inbox_id,omitempty"` + // Kind `site` measures websites and apps (page views, events, visitors). `api` watches an API: its + // host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An + // API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps + // allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, + // tracked props or consent features. The kind is fixed when the property is made. + Kind AnalyticsPropertyKind `json:"kind"` + // LastEventAt When an event was last stored, to the minute. - LastEventAt *time.Time `json:"last_event_at,omitempty"` - Name string `json:"name"` - PathRules []AnalyticsPathRule `json:"path_rules"` - Platforms []AnalyticsPlatform `json:"platforms"` + LastEventAt *time.Time `json:"last_event_at,omitempty"` + + // MonitorHourDays How many days an API's hourly rows are kept; daily rows follow `retention_months`. + MonitorHourDays int32 `json:"monitor_hour_days"` + Name string `json:"name"` + PathRules []AnalyticsPathRule `json:"path_rules"` + Platforms []AnalyticsPlatform `json:"platforms"` // PreviousKeyExpiresAt After a rotation, when the previous key stops working; absent when there is none. PreviousKeyExpiresAt *time.Time `json:"previous_key_expires_at,omitempty"` @@ -5659,23 +5882,35 @@ type AnalyticsProperty struct { // AnalyticsPropertyCreate defines model for AnalyticsPropertyCreate. type AnalyticsPropertyCreate struct { + ApdexMs *int32 `json:"apdex_ms,omitempty"` + // Audience `children` keeps location at country level and allows no consent features or tracked props. Audience *AnalyticsAudience `json:"audience,omitempty"` // Collection `anonymous` counts without consent: nothing is put on the visitor's device and no IP // address is stored. `consent` takes an event only when it carries `c.a: 1`, which the // script sends after `yuva.consent({analytics: true})`. - Collection *AnalyticsCollection `json:"collection,omitempty"` - ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` - Currency *string `json:"currency,omitempty"` - Domains []AnalyticsDomain `json:"domains"` - ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` - ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` - InboxId *uuid.UUID `json:"inbox_id,omitempty"` - Name string `json:"name"` - PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` - - // Platforms `[web]` when absent. + Collection *AnalyticsCollection `json:"collection,omitempty"` + ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` + Currency *string `json:"currency,omitempty"` + + // Domains 1 to 50 host names for a site, 0 to 50 for an API. + Domains []AnalyticsDomain `json:"domains"` + ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` + ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` + InboxId *uuid.UUID `json:"inbox_id,omitempty"` + + // Kind `site` measures websites and apps (page views, events, visitors). `api` watches an API: its + // host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An + // API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps + // allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, + // tracked props or consent features. The kind is fixed when the property is made. + Kind *AnalyticsPropertyKind `json:"kind,omitempty"` + MonitorHourDays *int32 `json:"monitor_hour_days,omitempty"` + Name string `json:"name"` + PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` + + // Platforms At least one for a site, `[web]` when absent; for an API `ios` and `android` only, none when absent. Platforms *[]AnalyticsPlatform `json:"platforms,omitempty"` // RetentionMonths 25 when absent; `null` keeps rollups forever. @@ -5692,6 +5927,13 @@ type AnalyticsPropertyDeletion struct { Name string `json:"name"` } +// AnalyticsPropertyKind `site` measures websites and apps (page views, events, visitors). `api` watches an API: its +// host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An +// API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps +// allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, +// tracked props or consent features. The kind is fixed when the property is made. +type AnalyticsPropertyKind string + // AnalyticsPropertyList defines model for AnalyticsPropertyList. type AnalyticsPropertyList struct { Items []AnalyticsProperty `json:"items"` @@ -5708,24 +5950,31 @@ type AnalyticsPropertyRef struct { // AnalyticsPropertyUpdate defines model for AnalyticsPropertyUpdate. type AnalyticsPropertyUpdate struct { + ApdexMs *int32 `json:"apdex_ms,omitempty"` + // Audience `children` keeps location at country level and allows no consent features or tracked props. Audience *AnalyticsAudience `json:"audience,omitempty"` // Collection `anonymous` counts without consent: nothing is put on the visitor's device and no IP // address is stored. `consent` takes an event only when it carries `c.a: 1`, which the // script sends after `yuva.consent({analytics: true})`. - Collection *AnalyticsCollection `json:"collection,omitempty"` - ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` - Currency *string `json:"currency,omitempty"` - Domains *[]AnalyticsDomain `json:"domains,omitempty"` - ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` - ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` + Collection *AnalyticsCollection `json:"collection,omitempty"` + ConsentFeatures *[]AnalyticsConsentFeature `json:"consent_features,omitempty"` + Currency *string `json:"currency,omitempty"` + + // Domains 1 to 50 host names for a site, 0 to 50 for an API. + Domains *[]AnalyticsDomain `json:"domains,omitempty"` + ExcludedIps *[]AnalyticsExcludedAddress `json:"excluded_ips,omitempty"` + ExcludedPaths *[]AnalyticsPathPattern `json:"excluded_paths,omitempty"` // InboxId `null` unlinks the property from its inbox. - InboxId nullable.Nullable[uuid.UUID] `json:"inbox_id,omitempty"` - Name *string `json:"name,omitempty"` - PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` - Platforms *[]AnalyticsPlatform `json:"platforms,omitempty"` + InboxId nullable.Nullable[uuid.UUID] `json:"inbox_id,omitempty"` + MonitorHourDays *int32 `json:"monitor_hour_days,omitempty"` + Name *string `json:"name,omitempty"` + PathRules *[]AnalyticsPathRule `json:"path_rules,omitempty"` + + // Platforms At least one for a site; for an API `ios` and `android` only. + Platforms *[]AnalyticsPlatform `json:"platforms,omitempty"` // RetentionMonths `null` keeps rollups forever. RetentionMonths nullable.Nullable[int32] `json:"retention_months,omitempty"` @@ -5753,6 +6002,14 @@ type AnalyticsQuery struct { // `support_rate` with pages of fewer than 100 page views after the others unless `order_by` // says otherwise. Days of the past come from the support rollup, today and days not yet rolled // up from the inbox itself. + // + // `api` is an API property's requests (`sendAnalyticsRequests`), and the only dataset an API + // property takes (`422 property_kind` otherwise, and for `api` on a site). It takes the API + // metrics of `AnalyticsMetric`, the API dimensions of `AnalyticsDimension` and filters on + // them, no goal, prop, segment, comparison or insights. Closed local days come from daily + // rows, closed hours from hourly rows and the rest from minute rows, which processes write + // every 10 seconds; `meta.served_from` is `rollups` when only daily and hourly rows answered. + // Rows are ordered by `requests` unless `order_by` says otherwise. Dataset *AnalyticsDataset `json:"dataset,omitempty"` Dimensions *[]AnalyticsDimension `json:"dimensions,omitempty"` Filters *[]AnalyticsFilter `json:"filters,omitempty"` @@ -5780,7 +6037,8 @@ type AnalyticsQuery struct { // Range Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, // `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and // `year` are the current calendar month and year to date; `last_month` the one before), or - // `from` and `to` (inclusive, at most 400 days apart). + // `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also + // be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. Range AnalyticsRange `json:"range"` // SegmentId A saved segment of the property, whose filters are added to `filters`. @@ -5790,7 +6048,8 @@ type AnalyticsQuery struct { // AnalyticsRange Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, // `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and // `year` are the current calendar month and year to date; `last_month` the one before), or -// `from` and `to` (inclusive, at most 400 days apart). +// `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also +// be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. type AnalyticsRange struct { From *openapi_types.Date `json:"from,omitempty"` Period *AnalyticsRangePeriod `json:"period,omitempty"` @@ -5892,6 +6151,74 @@ type AnalyticsRealtime struct { // AnalyticsReportFrequency defines model for AnalyticsReportFrequency. type AnalyticsReportFrequency string +// AnalyticsRequestRecord One request an API served. Give `route`, or `path` when the router knows no pattern. +type AnalyticsRequestRecord struct { + ApiVersion *string `json:"api_version,omitempty"` + + // Consumer Who called, as you name them, such as an API key's name or a client id; never a person's address (e-mail-shaped values become `(scrubbed)`). + Consumer *string `json:"consumer,omitempty"` + + // DurationUs How long the request took, in microseconds. + DurationUs int64 `json:"duration_us"` + + // Environment `production` when absent. + Environment *string `json:"environment,omitempty"` + + // Ip The caller's address, only to find its country; shortened on arrival, never stored. + Ip *string `json:"ip,omitempty"` + + // Method The request method; anything but the RFC 9110 methods and `PATCH` counts as `_OTHER`. + // + // Examples: GET + Method string `json:"method"` + + // Path The request path, when there is no `route`; only the pattern inferred from it is kept. + // + // Examples: /users/42 + Path *string `json:"path,omitempty"` + + // Release The service's release, for deploy markers. + Release *string `json:"release,omitempty"` + + // RequestSize Bytes of the request body. + RequestSize *int64 `json:"request_size,omitempty"` + + // ResponseSize Bytes of the response body. + ResponseSize *int64 `json:"response_size,omitempty"` + + // Route The route pattern that matched, starting with `/`, without method or host. + // + // Examples: /users/{id} + Route *string `json:"route,omitempty"` + + // SampleRate This record stands for this many requests, for a host that sends one in N successful + // requests. Only records with a status of 1 to 399 may be sampled. + SampleRate *int `json:"sample_rate,omitempty"` + + // Service The service that served the request. + Service *string `json:"service,omitempty"` + + // Status The response status, 100 to 599, or 0 when no response was sent. + Status int `json:"status"` + + // Time When the request started; now when absent. Records older than 72 hours are not counted. + Time *time.Time `json:"time,omitempty"` +} + +// AnalyticsRequestRecords defines model for AnalyticsRequestRecords. +type AnalyticsRequestRecords struct { + Records []AnalyticsRequestRecord `json:"records"` +} + +// AnalyticsRequestsResult defines model for AnalyticsRequestsResult. +type AnalyticsRequestsResult struct { + // Accepted The valid records, counted unless a rule of the operation dropped them. + Accepted int `json:"accepted"` + + // Errors The records left out as invalid, by their position in `records`. + Errors []AnalyticsEventError `json:"errors"` +} + // AnalyticsResult defines model for AnalyticsResult. type AnalyticsResult struct { Comparison *AnalyticsComparison `json:"comparison,omitempty"` @@ -5912,7 +6239,15 @@ type AnalyticsResult struct { // AnalyticsResultMeta defines model for AnalyticsResultMeta. type AnalyticsResultMeta struct { - From openapi_types.Date `json:"from"` + // CapsReached `dataset: api`: the caps the property has reached, so new values count as `(other)`: + // `routes` (1,000), `consumers` (10,000), `api_versions` (50), `app_versions` (200), + // `environments` (20), `services` (100 per workspace), `dimension_rows` (an hour of the + // range reached 50,000 consumer, country or app version rows). + CapsReached *[]AnalyticsResultMetaCapsReached `json:"caps_reached,omitempty"` + + // Estimated `dataset: api`: some requests of the range were sampled (`sample_rate`), so counts are estimates. + Estimated *bool `json:"estimated,omitempty"` + From openapi_types.Date `json:"from"` // RawWindowFrom The first day raw rows are kept for, which filters and breakdowns the rollups cannot answer need. RawWindowFrom openapi_types.Date `json:"raw_window_from"` @@ -5921,6 +6256,9 @@ type AnalyticsResultMeta struct { To openapi_types.Date `json:"to"` } +// AnalyticsResultMetaCapsReached defines model for AnalyticsResultMeta.CapsReached. +type AnalyticsResultMetaCapsReached string + // AnalyticsResultMetaServedFrom defines model for AnalyticsResultMeta.ServedFrom. type AnalyticsResultMetaServedFrom string @@ -6248,6 +6586,12 @@ type AnalyticsUsage struct { // Month Examples: 2026-10 Month string `json:"month"` + + // Requests API request records counted this month (`sendAnalyticsRequests`); absent when API monitoring is off. + Requests *int64 `json:"requests,omitempty"` + + // RequestsLimit The monthly cap on request records; absent when there is none. Records over it are not counted until the month ends. + RequestsLimit *int64 `json:"requests_limit,omitempty"` } // AnalyticsVisitPage defines model for AnalyticsVisitPage. @@ -6349,8 +6693,8 @@ type ApiKeyList struct { // Yuva. Owners and admins only; never offered to agents' tokens, refused for keys limited // to inboxes. // - `analytics:read`: analytics properties. -// - `analytics:write`: server-side analytics events; for API keys only, never offered to -// OAuth clients. +// - `analytics:write`: server-side analytics events and API request records; for API keys +// only, never offered to OAuth clients. // - `analytics:manage`: create, change and delete analytics properties, rotate their keys, // delete their data. Never offered to agents' tokens, refused for keys limited to inboxes. // @@ -14164,6 +14508,15 @@ type GetAnalyticsRealtimeParams struct { YuvaWorkspace *WorkspaceHeader `json:"Yuva-Workspace,omitempty"` } +// SendAnalyticsRequestsParams defines parameters for SendAnalyticsRequests. +type SendAnalyticsRequestsParams struct { + // ContentEncoding `gzip` when the body is compressed. + ContentEncoding *SendAnalyticsRequestsParamsContentEncoding `json:"Content-Encoding,omitempty"` +} + +// SendAnalyticsRequestsParamsContentEncoding defines parameters for SendAnalyticsRequests. +type SendAnalyticsRequestsParamsContentEncoding string + // GetAnalyticsRetentionParams defines parameters for GetAnalyticsRetention. type GetAnalyticsRetentionParams struct { // Grain `week` when absent. @@ -16312,6 +16665,9 @@ type UpdateAnalyticsGoalJSONRequestBody = AnalyticsGoalUpdate // QueryAnalyticsJSONRequestBody defines body for QueryAnalytics for application/json ContentType. type QueryAnalyticsJSONRequestBody = AnalyticsQuery +// SendAnalyticsRequestsJSONRequestBody defines body for SendAnalyticsRequests for application/json ContentType. +type SendAnalyticsRequestsJSONRequestBody = AnalyticsRequestRecords + // CreateAnalyticsSegmentJSONRequestBody defines body for CreateAnalyticsSegment for application/json ContentType. type CreateAnalyticsSegmentJSONRequestBody = AnalyticsSegmentCreate @@ -20912,11 +21268,12 @@ type ClientInterface interface { // CreateAnalyticsPropertyWithBody Create an analytics property // - // A website or app to measure. The answer carries its `public_key`, which pages and apps send - // events with. `inbox_id` links the property to an inbox: agents of that inbox can then read - // it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties - // (`403 property_limit`). A children's property (`audience: children`) cannot offer - // `consent_features` or `tracked_props`. + // A website or app to measure, or with `kind: api` an API to watch. The answer carries its + // `public_key`, which pages and apps send events with. `inbox_id` links the property to an + // inbox: agents of that inbox can then read it. A workspace has at most + // `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's + // property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API + // property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -20927,11 +21284,12 @@ type ClientInterface interface { // CreateAnalyticsProperty Create an analytics property // - // A website or app to measure. The answer carries its `public_key`, which pages and apps send - // events with. `inbox_id` links the property to an inbox: agents of that inbox can then read - // it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties - // (`403 property_limit`). A children's property (`audience: children`) cannot offer - // `consent_features` or `tracked_props`. + // A website or app to measure, or with `kind: api` an API to watch. The answer carries its + // `public_key`, which pages and apps send events with. `inbox_id` links the property to an + // inbox: agents of that inbox can then read it. A workspace has at most + // `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's + // property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API + // property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -21525,6 +21883,76 @@ type ClientInterface interface { // Corresponds with GET /v1/analytics/properties/{propertyId}/realtime (the `GetAnalyticsRealtime` operationId). GetAnalyticsRealtime(ctx context.Context, propertyId AnalyticsPropertyId, params *GetAnalyticsRealtimeParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // SendAnalyticsRequestsWithBody Send an API's request records + // + // The host of an API property reports the requests it served, 1 to 1,000 records a call, + // with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with + // `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key + // limited to inboxes reaches the properties of its inboxes. A site answers + // `422 property_kind`. Scope: `analytics:write`. + // + // A record names its route pattern (`/users/{id}`), as the router matched it. A record + // that can only give its `path` gets a pattern inferred from it: the query and fragment are + // cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other + // tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both + // cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that + // answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The + // property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 + // routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), + // environments (20) and services (100 per workspace) work the same way. + // + // Nothing else of a request is accepted: no body, header, query string, fragment or user + // agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country + // right away and never kept. Records are added up per minute and hour in the server's + // memory and written every 10 seconds, so a request shows in `queryAnalytics` + // (`dataset: api`) within about 15 seconds. + // + // Invalid records are left out on their own and listed in `errors`; the others are taken. + // Taken but not counted: records older than 72 hours, records over the property's + // `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap + // (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may + // (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. + // + // Takes any type of body and a specified content type. + // + // Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). + SendAnalyticsRequestsWithBody(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // SendAnalyticsRequests Send an API's request records + // + // The host of an API property reports the requests it served, 1 to 1,000 records a call, + // with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with + // `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key + // limited to inboxes reaches the properties of its inboxes. A site answers + // `422 property_kind`. Scope: `analytics:write`. + // + // A record names its route pattern (`/users/{id}`), as the router matched it. A record + // that can only give its `path` gets a pattern inferred from it: the query and fragment are + // cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other + // tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both + // cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that + // answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The + // property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 + // routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), + // environments (20) and services (100 per workspace) work the same way. + // + // Nothing else of a request is accepted: no body, header, query string, fragment or user + // agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country + // right away and never kept. Records are added up per minute and hour in the server's + // memory and written every 10 seconds, so a request shows in `queryAnalytics` + // (`dataset: api`) within about 15 seconds. + // + // Invalid records are left out on their own and listed in `errors`; the others are taken. + // Taken but not counted: records older than 72 hours, records over the property's + // `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap + // (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may + // (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. + // + // Takes a body of the `application/json` content type. + // + // Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). + SendAnalyticsRequests(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, body SendAnalyticsRequestsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetAnalyticsRetention How consented visitors come back // // Cohorts of the visitors with a persistent id (the `persistent` consent feature): each @@ -27392,11 +27820,12 @@ func (c *Client) ListAnalyticsProperties(ctx context.Context, params *ListAnalyt // CreateAnalyticsPropertyWithBody Create an analytics property // -// A website or app to measure. The answer carries its `public_key`, which pages and apps send -// events with. `inbox_id` links the property to an inbox: agents of that inbox can then read -// it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties -// (`403 property_limit`). A children's property (`audience: children`) cannot offer -// `consent_features` or `tracked_props`. +// A website or app to measure, or with `kind: api` an API to watch. The answer carries its +// `public_key`, which pages and apps send events with. `inbox_id` links the property to an +// inbox: agents of that inbox can then read it. A workspace has at most +// `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's +// property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API +// property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -27417,11 +27846,12 @@ func (c *Client) CreateAnalyticsPropertyWithBody(ctx context.Context, params *Cr // CreateAnalyticsProperty Create an analytics property // -// A website or app to measure. The answer carries its `public_key`, which pages and apps send -// events with. `inbox_id` links the property to an inbox: agents of that inbox can then read -// it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties -// (`403 property_limit`). A children's property (`audience: children`) cannot offer -// `consent_features` or `tracked_props`. +// A website or app to measure, or with `kind: api` an API to watch. The answer carries its +// `public_key`, which pages and apps send events with. `inbox_id` links the property to an +// inbox: agents of that inbox can then read it. A workspace has at most +// `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's +// property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API +// property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -28415,6 +28845,96 @@ func (c *Client) GetAnalyticsRealtime(ctx context.Context, propertyId AnalyticsP return c.Client.Do(req) } +// SendAnalyticsRequestsWithBody Send an API's request records +// +// The host of an API property reports the requests it served, 1 to 1,000 records a call, +// with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with +// `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key +// limited to inboxes reaches the properties of its inboxes. A site answers +// `422 property_kind`. Scope: `analytics:write`. +// +// A record names its route pattern (`/users/{id}`), as the router matched it. A record +// that can only give its `path` gets a pattern inferred from it: the query and fragment are +// cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other +// tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both +// cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that +// answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The +// property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 +// routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), +// environments (20) and services (100 per workspace) work the same way. +// +// Nothing else of a request is accepted: no body, header, query string, fragment or user +// agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country +// right away and never kept. Records are added up per minute and hour in the server's +// memory and written every 10 seconds, so a request shows in `queryAnalytics` +// (`dataset: api`) within about 15 seconds. +// +// Invalid records are left out on their own and listed in `errors`; the others are taken. +// Taken but not counted: records older than 72 hours, records over the property's +// `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap +// (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may +// (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. +// +// Takes any type of body and a specified content type. +// +// Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). +func (c *Client) SendAnalyticsRequestsWithBody(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewSendAnalyticsRequestsRequestWithBody(c.Server, propertyId, params, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// SendAnalyticsRequests Send an API's request records +// +// The host of an API property reports the requests it served, 1 to 1,000 records a call, +// with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with +// `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key +// limited to inboxes reaches the properties of its inboxes. A site answers +// `422 property_kind`. Scope: `analytics:write`. +// +// A record names its route pattern (`/users/{id}`), as the router matched it. A record +// that can only give its `path` gets a pattern inferred from it: the query and fragment are +// cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other +// tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both +// cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that +// answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The +// property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 +// routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), +// environments (20) and services (100 per workspace) work the same way. +// +// Nothing else of a request is accepted: no body, header, query string, fragment or user +// agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country +// right away and never kept. Records are added up per minute and hour in the server's +// memory and written every 10 seconds, so a request shows in `queryAnalytics` +// (`dataset: api`) within about 15 seconds. +// +// Invalid records are left out on their own and listed in `errors`; the others are taken. +// Taken but not counted: records older than 72 hours, records over the property's +// `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap +// (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may +// (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. +// +// Takes a body of the `application/json` content type. +// +// Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). +func (c *Client) SendAnalyticsRequests(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, body SendAnalyticsRequestsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewSendAnalyticsRequestsRequest(c.Server, propertyId, params, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // GetAnalyticsRetention How consented visitors come back // // Cohorts of the visitors with a persistent id (the `persistent` consent feature): each @@ -39613,6 +40133,68 @@ func NewGetAnalyticsRealtimeRequest(server string, propertyId AnalyticsPropertyI return req, nil } +// NewSendAnalyticsRequestsRequest calls the generic SendAnalyticsRequests builder with application/json body +func NewSendAnalyticsRequestsRequest(server string, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, body SendAnalyticsRequestsJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewSendAnalyticsRequestsRequestWithBody(server, propertyId, params, "application/json", bodyReader) +} + +// NewSendAnalyticsRequestsRequestWithBody constructs an http.Request for the SendAnalyticsRequests method, with any body, and a specified content type +func NewSendAnalyticsRequestsRequestWithBody(server string, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "propertyId", propertyId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: "uuid"}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/v1/analytics/properties/%s/requests", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + if params != nil { + + if params.ContentEncoding != nil { + var headerParam0 string + + headerParam0, err = runtime.StyleParamWithOptions("simple", false, "Content-Encoding", *params.ContentEncoding, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationHeader, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Encoding", headerParam0) + } + + } + + return req, nil +} + // NewGetAnalyticsRetentionRequest constructs an http.Request for the GetAnalyticsRetention method func NewGetAnalyticsRetentionRequest(server string, propertyId AnalyticsPropertyId, params *GetAnalyticsRetentionParams) (*http.Request, error) { var err error @@ -55193,11 +55775,12 @@ type ClientWithResponsesInterface interface { // CreateAnalyticsPropertyWithBodyWithResponse Create an analytics property // - // A website or app to measure. The answer carries its `public_key`, which pages and apps send - // events with. `inbox_id` links the property to an inbox: agents of that inbox can then read - // it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties - // (`403 property_limit`). A children's property (`audience: children`) cannot offer - // `consent_features` or `tracked_props`. + // A website or app to measure, or with `kind: api` an API to watch. The answer carries its + // `public_key`, which pages and apps send events with. `inbox_id` links the property to an + // inbox: agents of that inbox can then read it. A workspace has at most + // `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's + // property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API + // property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -55208,11 +55791,12 @@ type ClientWithResponsesInterface interface { // CreateAnalyticsPropertyWithResponse Create an analytics property // - // A website or app to measure. The answer carries its `public_key`, which pages and apps send - // events with. `inbox_id` links the property to an inbox: agents of that inbox can then read - // it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties - // (`403 property_limit`). A children's property (`audience: children`) cannot offer - // `consent_features` or `tracked_props`. + // A website or app to measure, or with `kind: api` an API to watch. The answer carries its + // `public_key`, which pages and apps send events with. `inbox_id` links the property to an + // inbox: agents of that inbox can then read it. A workspace has at most + // `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's + // property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API + // property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -55836,6 +56420,76 @@ type ClientWithResponsesInterface interface { // Corresponds with GET /v1/analytics/properties/{propertyId}/realtime (the `GetAnalyticsRealtime` operationId). GetAnalyticsRealtimeWithResponse(ctx context.Context, propertyId AnalyticsPropertyId, params *GetAnalyticsRealtimeParams, reqEditors ...RequestEditorFn) (*GetAnalyticsRealtimeResponse, error) + // SendAnalyticsRequestsWithBodyWithResponse Send an API's request records + // + // The host of an API property reports the requests it served, 1 to 1,000 records a call, + // with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with + // `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key + // limited to inboxes reaches the properties of its inboxes. A site answers + // `422 property_kind`. Scope: `analytics:write`. + // + // A record names its route pattern (`/users/{id}`), as the router matched it. A record + // that can only give its `path` gets a pattern inferred from it: the query and fragment are + // cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other + // tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both + // cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that + // answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The + // property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 + // routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), + // environments (20) and services (100 per workspace) work the same way. + // + // Nothing else of a request is accepted: no body, header, query string, fragment or user + // agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country + // right away and never kept. Records are added up per minute and hour in the server's + // memory and written every 10 seconds, so a request shows in `queryAnalytics` + // (`dataset: api`) within about 15 seconds. + // + // Invalid records are left out on their own and listed in `errors`; the others are taken. + // Taken but not counted: records older than 72 hours, records over the property's + // `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap + // (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may + // (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. + // + // Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). + SendAnalyticsRequestsWithBodyWithResponse(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*SendAnalyticsRequestsResponse, error) + + // SendAnalyticsRequestsWithResponse Send an API's request records + // + // The host of an API property reports the requests it served, 1 to 1,000 records a call, + // with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with + // `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key + // limited to inboxes reaches the properties of its inboxes. A site answers + // `422 property_kind`. Scope: `analytics:write`. + // + // A record names its route pattern (`/users/{id}`), as the router matched it. A record + // that can only give its `path` gets a pattern inferred from it: the query and fragment are + // cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other + // tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both + // cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that + // answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The + // property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 + // routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), + // environments (20) and services (100 per workspace) work the same way. + // + // Nothing else of a request is accepted: no body, header, query string, fragment or user + // agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country + // right away and never kept. Records are added up per minute and hour in the server's + // memory and written every 10 seconds, so a request shows in `queryAnalytics` + // (`dataset: api`) within about 15 seconds. + // + // Invalid records are left out on their own and listed in `errors`; the others are taken. + // Taken but not counted: records older than 72 hours, records over the property's + // `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap + // (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may + // (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. + // + // Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). + SendAnalyticsRequestsWithResponse(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, body SendAnalyticsRequestsJSONRequestBody, reqEditors ...RequestEditorFn) (*SendAnalyticsRequestsResponse, error) + // GetAnalyticsRetentionWithResponse How consented visitors come back // // Cohorts of the visitors with a persistent id (the `persistent` consent feature): each @@ -65301,6 +65955,89 @@ func (r GetAnalyticsRealtimeResponse) ContentType() string { return "" } +type SendAnalyticsRequestsResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON202 the response for an HTTP 202 `application/json` response + JSON202 *AnalyticsRequestsResult + // ApplicationproblemJSON400 the response for an HTTP 400 `application/problem+json` response + ApplicationproblemJSON400 *Problem + // ApplicationproblemJSON401 the response for an HTTP 401 `application/problem+json` response + ApplicationproblemJSON401 *Problem + // ApplicationproblemJSON403 the response for an HTTP 403 `application/problem+json` response + ApplicationproblemJSON403 *Problem + // ApplicationproblemJSON404 the response for an HTTP 404 `application/problem+json` response + ApplicationproblemJSON404 *Problem + // ApplicationproblemJSON422 the response for an HTTP 422 `application/problem+json` response + ApplicationproblemJSON422 *Problem + // ApplicationproblemJSON503 the response for an HTTP 503 `application/problem+json` response + ApplicationproblemJSON503 *Problem +} + +// GetJSON202 returns the response for an HTTP 202 `application/json` response +func (r SendAnalyticsRequestsResponse) GetJSON202() *AnalyticsRequestsResult { + return r.JSON202 +} + +// GetApplicationproblemJSON400 returns the response for an HTTP 400 `application/problem+json` response +func (r SendAnalyticsRequestsResponse) GetApplicationproblemJSON400() *Problem { + return r.ApplicationproblemJSON400 +} + +// GetApplicationproblemJSON401 returns the response for an HTTP 401 `application/problem+json` response +func (r SendAnalyticsRequestsResponse) GetApplicationproblemJSON401() *Problem { + return r.ApplicationproblemJSON401 +} + +// GetApplicationproblemJSON403 returns the response for an HTTP 403 `application/problem+json` response +func (r SendAnalyticsRequestsResponse) GetApplicationproblemJSON403() *Problem { + return r.ApplicationproblemJSON403 +} + +// GetApplicationproblemJSON404 returns the response for an HTTP 404 `application/problem+json` response +func (r SendAnalyticsRequestsResponse) GetApplicationproblemJSON404() *Problem { + return r.ApplicationproblemJSON404 +} + +// GetApplicationproblemJSON422 returns the response for an HTTP 422 `application/problem+json` response +func (r SendAnalyticsRequestsResponse) GetApplicationproblemJSON422() *Problem { + return r.ApplicationproblemJSON422 +} + +// GetApplicationproblemJSON503 returns the response for an HTTP 503 `application/problem+json` response +func (r SendAnalyticsRequestsResponse) GetApplicationproblemJSON503() *Problem { + return r.ApplicationproblemJSON503 +} + +// GetBody returns the raw response body bytes +func (r SendAnalyticsRequestsResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r SendAnalyticsRequestsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r SendAnalyticsRequestsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r SendAnalyticsRequestsResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type GetAnalyticsRetentionResponse struct { Body []byte HTTPResponse *http.Response @@ -82250,11 +82987,12 @@ func (c *ClientWithResponses) ListAnalyticsPropertiesWithResponse(ctx context.Co // CreateAnalyticsPropertyWithBodyWithResponse Create an analytics property // -// A website or app to measure. The answer carries its `public_key`, which pages and apps send -// events with. `inbox_id` links the property to an inbox: agents of that inbox can then read -// it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties -// (`403 property_limit`). A children's property (`audience: children`) cannot offer -// `consent_features` or `tracked_props`. +// A website or app to measure, or with `kind: api` an API to watch. The answer carries its +// `public_key`, which pages and apps send events with. `inbox_id` links the property to an +// inbox: agents of that inbox can then read it. A workspace has at most +// `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's +// property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API +// property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -82271,11 +83009,12 @@ func (c *ClientWithResponses) CreateAnalyticsPropertyWithBodyWithResponse(ctx co // CreateAnalyticsPropertyWithResponse Create an analytics property // -// A website or app to measure. The answer carries its `public_key`, which pages and apps send -// events with. `inbox_id` links the property to an inbox: agents of that inbox can then read -// it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties -// (`403 property_limit`). A children's property (`audience: children`) cannot offer -// `consent_features` or `tracked_props`. +// A website or app to measure, or with `kind: api` an API to watch. The answer carries its +// `public_key`, which pages and apps send events with. `inbox_id` links the property to an +// inbox: agents of that inbox can then read it. A workspace has at most +// `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's +// property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API +// property offers neither, and `audience` does not apply to it. // // Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. // @@ -83139,6 +83878,88 @@ func (c *ClientWithResponses) GetAnalyticsRealtimeWithResponse(ctx context.Conte return ParseGetAnalyticsRealtimeResponse(rsp) } +// SendAnalyticsRequestsWithBodyWithResponse Send an API's request records +// +// The host of an API property reports the requests it served, 1 to 1,000 records a call, +// with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with +// `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key +// limited to inboxes reaches the properties of its inboxes. A site answers +// `422 property_kind`. Scope: `analytics:write`. +// +// A record names its route pattern (`/users/{id}`), as the router matched it. A record +// that can only give its `path` gets a pattern inferred from it: the query and fragment are +// cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other +// tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both +// cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that +// answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The +// property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 +// routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), +// environments (20) and services (100 per workspace) work the same way. +// +// Nothing else of a request is accepted: no body, header, query string, fragment or user +// agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country +// right away and never kept. Records are added up per minute and hour in the server's +// memory and written every 10 seconds, so a request shows in `queryAnalytics` +// (`dataset: api`) within about 15 seconds. +// +// Invalid records are left out on their own and listed in `errors`; the others are taken. +// Taken but not counted: records older than 72 hours, records over the property's +// `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap +// (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may +// (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. +// +// Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). +func (c *ClientWithResponses) SendAnalyticsRequestsWithBodyWithResponse(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*SendAnalyticsRequestsResponse, error) { + rsp, err := c.SendAnalyticsRequestsWithBody(ctx, propertyId, params, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseSendAnalyticsRequestsResponse(rsp) +} + +// SendAnalyticsRequestsWithResponse Send an API's request records +// +// The host of an API property reports the requests it served, 1 to 1,000 records a call, +// with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with +// `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key +// limited to inboxes reaches the properties of its inboxes. A site answers +// `422 property_kind`. Scope: `analytics:write`. +// +// A record names its route pattern (`/users/{id}`), as the router matched it. A record +// that can only give its `path` gets a pattern inferred from it: the query and fragment are +// cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other +// tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both +// cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that +// answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The +// property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 +// routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), +// environments (20) and services (100 per workspace) work the same way. +// +// Nothing else of a request is accepted: no body, header, query string, fragment or user +// agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country +// right away and never kept. Records are added up per minute and hour in the server's +// memory and written every 10 seconds, so a request shows in `queryAnalytics` +// (`dataset: api`) within about 15 seconds. +// +// Invalid records are left out on their own and listed in `errors`; the others are taken. +// Taken but not counted: records older than 72 hours, records over the property's +// `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap +// (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may +// (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. +// +// Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /v1/analytics/properties/{propertyId}/requests (the `SendAnalyticsRequests` operationId). +func (c *ClientWithResponses) SendAnalyticsRequestsWithResponse(ctx context.Context, propertyId AnalyticsPropertyId, params *SendAnalyticsRequestsParams, body SendAnalyticsRequestsJSONRequestBody, reqEditors ...RequestEditorFn) (*SendAnalyticsRequestsResponse, error) { + rsp, err := c.SendAnalyticsRequests(ctx, propertyId, params, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseSendAnalyticsRequestsResponse(rsp) +} + // GetAnalyticsRetentionWithResponse How consented visitors come back // // Cohorts of the visitors with a persistent id (the `persistent` consent feature): each @@ -93371,6 +94192,74 @@ func ParseGetAnalyticsRealtimeResponse(rsp *http.Response) (*GetAnalyticsRealtim return response, nil } +// ParseSendAnalyticsRequestsResponse parses an HTTP response from a SendAnalyticsRequestsWithResponse call +func ParseSendAnalyticsRequestsResponse(rsp *http.Response) (*SendAnalyticsRequestsResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &SendAnalyticsRequestsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 202: + var dest AnalyticsRequestsResult + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON202 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Problem + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest Problem + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest Problem + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON403 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Problem + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 422: + var dest Problem + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON422 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 503: + var dest Problem + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON503 = &dest + + } + + return response, nil +} + // ParseGetAnalyticsRetentionResponse parses an HTTP response from a GetAnalyticsRetentionWithResponse call func ParseGetAnalyticsRetentionResponse(rsp *http.Response) (*GetAnalyticsRetentionResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) From a2f178eca72b4ba48682c8029c48df682f2b666a Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:40:13 +0200 Subject: [PATCH 04/69] chore(sdk/js): regenerate schema.gen.ts for API properties and request records Co-Authored-By: Claude Opus 5.5 --- sdk/js/src/schema.gen.ts | 285 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 269 insertions(+), 16 deletions(-) diff --git a/sdk/js/src/schema.gen.ts b/sdk/js/src/schema.gen.ts index eb34c7a6..55d5039e 100644 --- a/sdk/js/src/schema.gen.ts +++ b/sdk/js/src/schema.gen.ts @@ -4619,11 +4619,12 @@ export interface paths { put?: never; /** * Create an analytics property - * @description A website or app to measure. The answer carries its `public_key`, which pages and apps send - * events with. `inbox_id` links the property to an inbox: agents of that inbox can then read - * it. A workspace has at most `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties - * (`403 property_limit`). A children's property (`audience: children`) cannot offer - * `consent_features` or `tracked_props`. + * @description A website or app to measure, or with `kind: api` an API to watch. The answer carries its + * `public_key`, which pages and apps send events with. `inbox_id` links the property to an + * inbox: agents of that inbox can then read it. A workspace has at most + * `YUVA_ANALYTICS_PROPERTIES_PER_WORKSPACE` properties (`403 property_limit`). A children's + * property (`audience: children`) cannot offer `consent_features` or `tracked_props`; an API + * property offers neither, and `audience` does not apply to it. * * Owners, admins and API keys without an inbox limit. Scope: `analytics:manage`. */ @@ -4840,6 +4841,52 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/analytics/properties/{propertyId}/requests": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Send an API's request records + * @description The host of an API property reports the requests it served, 1 to 1,000 records a call, + * with `sdk/go/analytics` (`Middleware`) or any HTTP client. The body may be sent with + * `Content-Encoding: gzip`. API keys only (`403 forbidden` for members' sessions); a key + * limited to inboxes reaches the properties of its inboxes. A site answers + * `422 property_kind`. Scope: `analytics:write`. + * + * A record names its route pattern (`/users/{id}`), as the router matched it. A record + * that can only give its `path` gets a pattern inferred from it: the query and fragment are + * cut, and segments that are numbers, UUIDs, dates, hex of 16 or more characters, other + * tokens of 16 or more letters, digits, `-` and `_` with at least two digits or with both + * cases and a digit, e-mail-shaped or longer than 40 characters become `{id}`. A path that + * answered 404 or 405 is counted as `(unmatched)`, so scanners never make routes. The + * property's `path_rules` then map a path or route onto a pattern of their own. Past 1,000 + * routes a property counts new ones as `(other)`; consumers (10,000), API versions (50), + * environments (20) and services (100 per workspace) work the same way. + * + * Nothing else of a request is accepted: no body, header, query string, fragment or user + * agent. `ip`, when given, is shortened (IPv4 /24, IPv6 /48) and looked up for its country + * right away and never kept. Records are added up per minute and hour in the server's + * memory and written every 10 seconds, so a request shows in `queryAnalytics` + * (`dataset: api`) within about 15 seconds. + * + * Invalid records are left out on their own and listed in `errors`; the others are taken. + * Taken but not counted: records older than 72 hours, records over the property's + * `YUVA_MONITOR_RECORDS_PER_MINUTE` or the workspace's monthly cap + * (`YUVA_MONITOR_MONTHLY_REQUESTS`). When the server holds as many open rows as it may + * (`YUVA_MONITOR_BUFFER_KEYS`) it answers `503` with `Retry-After`. + */ + post: operations["sendAnalyticsRequests"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/analytics/properties/{propertyId}/retention": { parameters: { query?: never; @@ -7299,8 +7346,8 @@ export interface components { * Yuva. Owners and admins only; never offered to agents' tokens, refused for keys limited * to inboxes. * - `analytics:read`: analytics properties. - * - `analytics:write`: server-side analytics events; for API keys only, never offered to - * OAuth clients. + * - `analytics:write`: server-side analytics events and API request records; for API keys + * only, never offered to OAuth clients. * - `analytics:manage`: create, change and delete analytics properties, rotate their keys, * delete their data. Never offered to agents' tokens, refused for keys limited to inboxes. * @@ -12704,10 +12751,20 @@ export interface components { */ AnalyticsExcludedAddress: string; AnalyticsTrackedProp: string; - /** @description A website or app the workspace measures. */ + /** + * @description `site` measures websites and apps (page views, events, visitors). `api` watches an API: its + * host sends a record per request (`sendAnalyticsRequests`) and `dataset: api` queries them. An + * API property's `domains` (0 to 50) are the API's host names and its `platforms` the apps + * allowed to send app-side records (`ios`, `android`, or none); it takes no goals, funnels, + * tracked props or consent features. The kind is fixed when the property is made. + * @enum {string} + */ + AnalyticsPropertyKind: "site" | "api"; + /** @description A website, app or API the workspace measures. */ AnalyticsProperty: { /** Format: uuid */ id: string; + kind: components["schemas"]["AnalyticsPropertyKind"]; /** * Format: uuid * @description The linked inbox, whose agents can read the property; absent when none. @@ -12742,6 +12799,16 @@ export interface components { * @description Rollups older than this are deleted; absent when they are kept forever. */ retention_months?: number; + /** + * Format: int32 + * @description An API's Apdex threshold T in milliseconds; a request is satisfied within T and tolerated within 4T. + */ + apdex_ms: number; + /** + * Format: int32 + * @description How many days an API's hourly rows are kept; daily rows follow `retention_months`. + */ + monitor_hour_days: number; /** * Format: int64 * @description Distinct visitors so far today, in the property's time zone. @@ -12763,12 +12830,24 @@ export interface components { updated_at: string; }; AnalyticsPropertyCreate: { + kind?: components["schemas"]["AnalyticsPropertyKind"]; name: string; /** Format: uuid */ inbox_id?: string; + /** @description 1 to 50 host names for a site, 0 to 50 for an API. */ domains: components["schemas"]["AnalyticsDomain"][]; - /** @description `[web]` when absent. */ + /** @description At least one for a site, `[web]` when absent; for an API `ios` and `android` only, none when absent. */ platforms?: components["schemas"]["AnalyticsPlatform"][]; + /** + * Format: int32 + * @default 500 + */ + apdex_ms: number; + /** + * Format: int32 + * @default 90 + */ + monitor_hour_days: number; /** @description The linked inbox's time zone when absent, else `UTC`. */ timezone?: string; /** @default EUR */ @@ -12793,8 +12872,14 @@ export interface components { * @description `null` unlinks the property from its inbox. */ inbox_id?: string | null; + /** @description 1 to 50 host names for a site, 0 to 50 for an API. */ domains?: components["schemas"]["AnalyticsDomain"][]; + /** @description At least one for a site; for an API `ios` and `android` only. */ platforms?: components["schemas"]["AnalyticsPlatform"][]; + /** Format: int32 */ + apdex_ms?: number; + /** Format: int32 */ + monitor_hour_days?: number; timezone?: string; currency?: string; collection?: components["schemas"]["AnalyticsCollection"]; @@ -12829,6 +12914,16 @@ export interface components { * @description When this month's events reached the cap; later events are dropped until the month ends. */ limit_reached_at?: string; + /** + * Format: int64 + * @description API request records counted this month (`sendAnalyticsRequests`); absent when API monitoring is off. + */ + requests?: number; + /** + * Format: int64 + * @description The monthly cap on request records; absent when there is none. Records over it are not counted until the month ends. + */ + requests_limit?: number; }; AnalyticsPropertyList: { items: components["schemas"]["AnalyticsProperty"][]; @@ -12867,9 +12962,16 @@ export interface components { * `ratings_down` (page ratings), `helpful_rate` (percent of ratings that are `up`), * `discussions` (discussion threads started on it) and `support_rate` (conversations per * 1,000 page views), next to `visitors` and `pageviews`. + * + * With `dataset: api` (an API property) only: `requests`, `requests_per_minute` (requests + * per minute of the range or bucket, up to now), `errors` (status 500 to 599, or 0 when no + * response was sent), `error_rate` and `client_error_rate` (percent of requests), + * `client_errors` (status 400 to 499), `p50`, `p95` and `p99` (the duration in milliseconds + * that this share of requests stayed within, from sketches accurate to 1.1%) and + * `avg_duration` (milliseconds). * @enum {string} */ - AnalyticsMetric: "visitors" | "visits" | "pageviews" | "views_per_visit" | "bounce_rate" | "visit_duration" | "time_on_page" | "scroll_depth" | "events" | "conversions" | "conversion_rate" | "completions" | "revenue" | "conversations" | "feedback" | "ratings_up" | "ratings_down" | "helpful_rate" | "discussions" | "support_rate"; + AnalyticsMetric: "visitors" | "visits" | "pageviews" | "views_per_visit" | "bounce_rate" | "visit_duration" | "time_on_page" | "scroll_depth" | "events" | "conversions" | "conversion_rate" | "completions" | "revenue" | "conversations" | "feedback" | "ratings_up" | "ratings_down" | "helpful_rate" | "discussions" | "support_rate" | "requests" | "requests_per_minute" | "errors" | "error_rate" | "client_errors" | "client_error_rate" | "p50" | "p95" | "p99" | "avg_duration"; /** * @description A value of a visit or page. `page` is a path that was viewed (pages are counted by path; * `hostname` tells sites apart), `entry_page` and `exit_page` the first and last of a visit. @@ -12887,9 +12989,15 @@ export interface components { * for events without it: as * a filter of visits they keep the visits with such an event; in an `event` or `prop` * breakdown they keep the matching events. + * + * With `dataset: api`: `route` is the route pattern (`/users/{id}`), `(unmatched)` for + * requests that answered 404 or 405 without one, or `(other)` past the property's 1,000 + * routes; `method` an HTTP method or `_OTHER`; `status` the status code (`0` when no response + * was sent); `status_class` `1xx` to `5xx`, or `none` for status 0; `environment` the record's + * environment (`production` by default, `(other)` past 20). * @enum {string} */ - AnalyticsFilterDimension: "page" | "entry_page" | "exit_page" | "hostname" | "channel" | "source" | "referrer" | "utm_source" | "utm_medium" | "utm_campaign" | "utm_term" | "utm_content" | "country" | "region" | "city" | "device" | "browser" | "browser_version" | "os" | "os_version" | "screen_size" | "language" | "platform" | "app_version" | "visitor_type" | "event" | "prop"; + AnalyticsFilterDimension: "page" | "entry_page" | "exit_page" | "hostname" | "channel" | "source" | "referrer" | "utm_source" | "utm_medium" | "utm_campaign" | "utm_term" | "utm_content" | "country" | "region" | "city" | "device" | "browser" | "browser_version" | "os" | "os_version" | "screen_size" | "language" | "platform" | "app_version" | "visitor_type" | "event" | "prop" | "route" | "method" | "status" | "status_class" | "environment"; /** * @description A filter dimension (`AnalyticsFilterDimension`), or a time grain: `time:hour` (local hours * of the property's time zone; only alone, and only with `visitors`, `visits`, `pageviews` and @@ -12898,9 +13006,14 @@ export interface components { * range is present. `event` breaks custom and built-in events down by name, `prop` by the * value of the event prop the query's `prop` names; either takes at most one other dimension, * a time grain other than `time:hour` or a visit dimension. + * + * With `dataset: api`, two of `route`, `method`, `status`, `status_class`, `environment` and + * one grain: `time:minute` (minutes, `2026-10-12T14:05` in the property's time zone, inside + * the last `YUVA_MONITOR_MINUTE_DAYS` days and at most 1,440 of them), `time:hour` (local + * hours) or `time:day`. * @enum {string} */ - AnalyticsDimension: "time:hour" | "time:day" | "time:week" | "time:month" | "page" | "entry_page" | "exit_page" | "hostname" | "channel" | "source" | "referrer" | "utm_source" | "utm_medium" | "utm_campaign" | "utm_term" | "utm_content" | "country" | "region" | "city" | "device" | "browser" | "browser_version" | "os" | "os_version" | "screen_size" | "language" | "platform" | "app_version" | "visitor_type" | "event" | "prop"; + AnalyticsDimension: "time:minute" | "time:hour" | "time:day" | "time:week" | "time:month" | "page" | "entry_page" | "exit_page" | "hostname" | "channel" | "source" | "referrer" | "utm_source" | "utm_medium" | "utm_campaign" | "utm_term" | "utm_content" | "country" | "region" | "city" | "device" | "browser" | "browser_version" | "os" | "os_version" | "screen_size" | "language" | "platform" | "app_version" | "visitor_type" | "event" | "prop" | "route" | "method" | "status" | "status_class" | "environment"; /** * @description `is` and `is_not` compare whole values, `contains` and `not_contains` parts of them (case * does not matter), `matches` patterns: on paths `*` is one segment and `**` any number of @@ -12919,11 +13032,12 @@ export interface components { * @description Days of the property's time zone: a `period` ending today (`today`, `yesterday`, `7d`, * `28d`, `30d`, `90d`, `6mo`, `12mo` count back from today, today included; `month` and * `year` are the current calendar month and year to date; `last_month` the one before), or - * `from` and `to` (inclusive, at most 400 days apart). + * `from` and `to` (inclusive, at most 400 days apart). With `dataset: api` a period can also + * be the last `15m`, `1h`, `6h` or `24h` up to the current minute, included. */ AnalyticsRange: { /** @enum {string} */ - period?: "today" | "yesterday" | "7d" | "28d" | "30d" | "90d" | "6mo" | "12mo" | "month" | "last_month" | "year"; + period?: "today" | "yesterday" | "7d" | "28d" | "30d" | "90d" | "6mo" | "12mo" | "month" | "last_month" | "year" | "15m" | "1h" | "6h" | "24h"; /** Format: date */ from?: string; /** Format: date */ @@ -12961,9 +13075,17 @@ export interface components { * `support_rate` with pages of fewer than 100 page views after the others unless `order_by` * says otherwise. Days of the past come from the support rollup, today and days not yet rolled * up from the inbox itself. + * + * `api` is an API property's requests (`sendAnalyticsRequests`), and the only dataset an API + * property takes (`422 property_kind` otherwise, and for `api` on a site). It takes the API + * metrics of `AnalyticsMetric`, the API dimensions of `AnalyticsDimension` and filters on + * them, no goal, prop, segment, comparison or insights. Closed local days come from daily + * rows, closed hours from hourly rows and the rest from minute rows, which processes write + * every 10 seconds; `meta.served_from` is `rollups` when only daily and hourly rows answered. + * Rows are ordered by `requests` unless `order_by` says otherwise. * @enum {string} */ - AnalyticsDataset: "traffic" | "support"; + AnalyticsDataset: "traffic" | "support" | "api"; AnalyticsQuery: { dataset?: components["schemas"]["AnalyticsDataset"]; metrics: components["schemas"]["AnalyticsMetric"][]; @@ -13044,6 +13166,26 @@ export interface components { discussions?: number; /** Format: double */ support_rate?: number; + /** Format: double */ + requests?: number; + /** Format: double */ + requests_per_minute?: number; + /** Format: double */ + errors?: number; + /** Format: double */ + error_rate?: number; + /** Format: double */ + client_errors?: number; + /** Format: double */ + client_error_rate?: number; + /** Format: double */ + p50?: number; + /** Format: double */ + p95?: number; + /** Format: double */ + p99?: number; + /** Format: double */ + avg_duration?: number; }; AnalyticsRevenue: { /** @description ISO 4217 code. */ @@ -13097,6 +13239,15 @@ export interface components { * @description The first day raw rows are kept for, which filters and breakdowns the rollups cannot answer need. */ raw_window_from: string; + /** @description `dataset: api`: some requests of the range were sampled (`sample_rate`), so counts are estimates. */ + estimated?: boolean; + /** + * @description `dataset: api`: the caps the property has reached, so new values count as `(other)`: + * `routes` (1,000), `consumers` (10,000), `api_versions` (50), `app_versions` (200), + * `environments` (20), `services` (100 per workspace), `dimension_rows` (an hour of the + * range reached 50,000 consumer, country or app version rows). + */ + caps_reached?: ("routes" | "consumers" | "api_versions" | "app_versions" | "environments" | "services" | "dimension_rows")[]; }; AnalyticsResult: { /** @description Empty when the query has no dimensions. */ @@ -13327,6 +13478,72 @@ export interface components { /** @description An identity token as in `AnalyticsConsent.j`, with `persistent_id` and `consent: true`, on a property that offers `identify`. */ identity_token?: string; }; + AnalyticsRequestRecords: { + records: components["schemas"]["AnalyticsRequestRecord"][]; + }; + /** @description One request an API served. Give `route`, or `path` when the router knows no pattern. */ + AnalyticsRequestRecord: { + /** + * @description The request method; anything but the RFC 9110 methods and `PATCH` counts as `_OTHER`. + * @example GET + */ + method: string; + /** + * @description The route pattern that matched, starting with `/`, without method or host. + * @example /users/{id} + */ + route?: string; + /** + * @description The request path, when there is no `route`; only the pattern inferred from it is kept. + * @example /users/42 + */ + path?: string; + /** @description The response status, 100 to 599, or 0 when no response was sent. */ + status: number; + /** + * Format: int64 + * @description How long the request took, in microseconds. + */ + duration_us: number; + /** + * Format: int64 + * @description Bytes of the request body. + */ + request_size?: number; + /** + * Format: int64 + * @description Bytes of the response body. + */ + response_size?: number; + /** + * Format: date-time + * @description When the request started; now when absent. Records older than 72 hours are not counted. + */ + time?: string; + /** @description Who called, as you name them, such as an API key's name or a client id; never a person's address (e-mail-shaped values become `(scrubbed)`). */ + consumer?: string; + api_version?: string; + /** @description `production` when absent. */ + environment?: string; + /** @description The service that served the request. */ + service?: string; + /** @description The service's release, for deploy markers. */ + release?: string; + /** @description The caller's address, only to find its country; shortened on arrival, never stored. */ + ip?: string; + /** + * @description This record stands for this many requests, for a host that sends one in N successful + * requests. Only records with a status of 1 to 399 may be sampled. + * @default 1 + */ + sample_rate: number; + }; + AnalyticsRequestsResult: { + /** @description The valid records, counted unless a rule of the operation dropped them. */ + accepted: number; + /** @description The records left out as invalid, by their position in `records`. */ + errors: components["schemas"]["AnalyticsEventError"][]; + }; AnalyticsEventsResult: { /** @description The valid events, stored unless a rule of the operation dropped them. */ accepted: number; @@ -22180,7 +22397,8 @@ export interface operations { /** * @description `needs_raw_window`: the query needs raw rows for days that are no longer kept; * `raw_window_from` is the first day it can serve. `range_too_large`: the range is longer - * than 400 days. + * than 400 days, or with `time:minute` more than 1,440 minutes or older than the minute + * rows. `property_kind`: the dataset does not fit the property's kind. */ 422: { headers: { @@ -22253,6 +22471,41 @@ export interface operations { 503: components["responses"]["Problem"]; }; }; + sendAnalyticsRequests: { + parameters: { + query?: never; + header?: { + /** @description `gzip` when the body is compressed. */ + "Content-Encoding"?: "gzip" | "identity"; + }; + path: { + propertyId: components["parameters"]["AnalyticsPropertyId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AnalyticsRequestRecords"]; + }; + }; + responses: { + /** @description Taken; counted unless one of the rules above dropped them. */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AnalyticsRequestsResult"]; + }; + }; + 400: components["responses"]["Problem"]; + 401: components["responses"]["Problem"]; + 403: components["responses"]["Problem"]; + 404: components["responses"]["Problem"]; + 422: components["responses"]["Problem"]; + 503: components["responses"]["Problem"]; + }; + }; getAnalyticsRetention: { parameters: { query?: { From afc19d658bfe98c725a1c09cfe147ff3c8251a1e Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:40:13 +0200 Subject: [PATCH 05/69] test(api): API monitoring load test at 300M requests a month Writes 30 days of folded rows for one property with 500 routes, drives the records endpoint at 2,000 records a second from 5,000 consumers and times ingest, visibility and the overview queries. Runs with YUVA_TEST_MONITOR_LOAD set. Refs #64 Co-Authored-By: Claude Opus 5.5 --- api/internal/api/monitor_load_test.go | 344 ++++++++++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 api/internal/api/monitor_load_test.go diff --git a/api/internal/api/monitor_load_test.go b/api/internal/api/monitor_load_test.go new file mode 100644 index 00000000..c8a59d43 --- /dev/null +++ b/api/internal/api/monitor_load_test.go @@ -0,0 +1,344 @@ +package api_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math" + "math/rand/v2" + "net/http" + "os" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + "uuid" + + "github.com/jackc/pgx/v5" + + "github.com/productdevbook/yuva/api/internal/api" + "github.com/productdevbook/yuva/api/internal/monitor" + "github.com/productdevbook/yuva/api/internal/store" +) + +// The API monitoring load test runs only when YUVA_TEST_MONITOR_LOAD is set; YUVA_TEST_MONITOR_REPORT, +// when set, is a file the report is written to. It writes 30 days of folded rows for 300M requests a +// month on one property (500 routes) and then drives the records endpoint in bursts of +// YUVA_TEST_MONITOR_RATE records a second (2000) for YUVA_TEST_MONITOR_SECONDS (30), from 5,000 +// consumers, with the aggregator flushing every 10 seconds. + +const ( + loadRoutes = 500 + loadConsumers = 5000 + loadMonthly = 300_000_000 + loadDays = 30 + loadStatusesPer = 4 +) + +func TestMonitorLoad(t *testing.T) { + if os.Getenv("YUVA_TEST_MONITOR_LOAD") == "" { + t.Skip("YUVA_TEST_MONITOR_LOAD is not set") + } + h := newHarnessWith(t, func(d *api.Deps) { + d.Analytics.Monitor.RecordsPerMinute = 1 << 30 + }) + ctx := context.Background() + today := monitor.LocalDay(time.Now(), time.UTC) + now := today.Add(16*time.Hour + 30*time.Second) + h.clock.Set(now) + rep := &loadReport{} + rep.add(t, "# API monitoring load test, %s, Go %s, %d CPUs", time.Now().UTC().Format(time.DateOnly), runtime.Version(), runtime.NumCPU()) + tm := newTeam(t, h) + id := createAPIProperty(tm.owner, map[string]any{"name": "Load API"}) + ws, prop := uuid.MustParse(tm.ws), uuid.MustParse(id) + st := &monitor.PGStore{Pool: h.st.Pool} + + start := time.Now() + rows := writeMonitorHistory(t, h, st, ws, prop, today, now) + rep.add(t, "## History: %d requests a month, %d days and today's hours, %d rows, written in %v", loadMonthly, loadDays-1, rows, time.Since(start).Round(time.Second)) + + backend := tm.apiKey(h) + loadMonitorIngest(t, h, rep, backend.bearer, id) + + end := now.Truncate(time.Minute).Add(time.Minute) + views := []struct { + name string + q monitor.Query + target time.Duration + }{ + {"Overview, 30 days by day: requests, error rates, p95", monitor.Query{Start: monitor.DayStart(today.AddDate(0, 0, 1-loadDays), time.UTC), End: monitor.DayStart(today.AddDate(0, 0, 1), time.UTC), + Days: true, Dimensions: []string{"time:day"}, Metrics: []string{"requests", "error_rate", "client_error_rate", "p95"}}, time.Second}, + {"Routes, 30 days: requests, error rate, p95", monitor.Query{Start: monitor.DayStart(today.AddDate(0, 0, 1-loadDays), time.UTC), End: monitor.DayStart(today.AddDate(0, 0, 1), time.UTC), + Days: true, Dimensions: []string{"route"}, Metrics: []string{"requests", "error_rate", "p50", "p95", "p99"}}, time.Second}, + {"Last 60 minutes by minute", monitor.Query{Start: end.Add(-time.Hour), End: end, Dimensions: []string{"time:minute"}, + Metrics: []string{"requests", "error_rate", "p95"}}, time.Second}, + {"A route's 30 days by status", monitor.Query{Start: monitor.DayStart(today.AddDate(0, 0, 1-loadDays), time.UTC), End: monitor.DayStart(today.AddDate(0, 0, 1), time.UTC), + Days: true, Dimensions: []string{"status", "time:day"}, Metrics: []string{"requests", "p95"}, + Filters: []monitor.Filter{{Dimension: "route", Op: "is", Values: []string{"/v1/things7/{id}"}}}}, time.Second}, + } + rep.add(t, "| View | p50 | p95 | target (p95) |") + rep.add(t, "|---|---|---|---|") + for _, v := range views { + q := v.q + q.WorkspaceID, q.PropertyID, q.Location, q.Now, q.MinuteFrom = ws, prop, time.UTC, now, today.AddDate(0, 0, -7) + var times []time.Duration + var res monitor.Result + for range 20 { + s := time.Now() + var err error + if res, err = st.Query(ctx, q); err != nil { + t.Fatal(err) + } + times = append(times, time.Since(s)) + } + p50, p95 := quantile(times, 0.5), quantile(times, 0.95) + rep.add(t, "| %s (%d rows, %s) | %v | %v | < %v |", v.name, len(res.Rows), res.ServedFrom, p50.Round(time.Millisecond), p95.Round(time.Millisecond), v.target) + if p95 > v.target { + t.Errorf("%s: p95 %v over the target %v", v.name, p95, v.target) + } + } + var size int64 + if err := h.st.Pool.QueryRow(ctx, `SELECT coalesce(sum(pg_total_relation_size(c.oid)), 0)::bigint FROM pg_class c + WHERE c.relkind = 'r' AND c.relname LIKE 'monitor\_%'`).Scan(&size); err != nil { + t.Fatal(err) + } + rep.add(t, "- monitor tables %.0f MiB with indexes", float64(size)/(1<<20)) + if path := os.Getenv("YUVA_TEST_MONITOR_REPORT"); path != "" { + if err := os.WriteFile(path, []byte(strings.Join(rep.lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + if _, err := h.st.Pool.Exec(ctx, "DELETE FROM monitor_minute WHERE workspace_id = $1", ws); err != nil { + t.Error(err) + } + if err := h.st.DeleteMonitorRollups(ctx, store.DeleteMonitorRollupsParams{WorkspaceID: ws, PropertyID: prop}); err != nil { + t.Error(err) + } +} + +func loadRoute(i int) string { return fmt.Sprintf("/v1/things%d/{id}", i) } + +func loadDuration(r *rand.Rand, route int) int64 { + return int64(math.Exp(r.NormFloat64()*0.8+8+float64(route%7)*0.3)) + 1 +} + +// writeMonitorHistory writes folded days before today and folded hours of today up to the last +// full hour, as the folds would have, and returns how many rows it wrote. +func writeMonitorHistory(t *testing.T, h *harness, st *monitor.PGStore, ws, prop uuid.UUID, today, now time.Time) int { + t.Helper() + ctx := context.Background() + if err := st.EnsureHourPartitions(ctx, today.AddDate(0, 0, -loadDays), today); err != nil { + t.Fatal(err) + } + var routes []monitor.NameRequest + for i := range loadRoutes { + routes = append(routes, monitor.NameRequest{Kind: monitor.KindRoute, Value: loadRoute(i)}) + } + routes = append(routes, monitor.NameRequest{Kind: monitor.KindEnvironment, Value: monitor.DefaultEnvironment}) + ids, err := st.Names(ctx, ws, prop, routes, now) + if err != nil { + t.Fatal(err) + } + env := ids[len(ids)-1] + statuses := []int16{200, 201, 404, 500} + shares := []float64{0.9, 0.06, 0.03, 0.01} + perKeyDay := float64(loadMonthly) / 30 / float64(loadRoutes) + r := rand.New(rand.NewPCG(3, 4)) + agg := func(route int, n int64, status int16) monitor.Agg { + a := monitor.NewAgg() + samples := min(n, 400) + for range samples { + rec := monitor.Record{DurationUS: loadDuration(r, route), Status: status, RequestBytes: 120, ResponseBytes: int64(500 + r.IntN(2000)), Weight: n / samples} + a.Add(&rec) + } + return a + } + cols := []string{"workspace_id", "property_id", "day", "environment", "service", "route", "method", "status", "api_version", "source", "platform", + "requests", "errors", "client_errors", "sampled", "duration_sum", "duration_min", "duration_max", "durations", "request_bytes", "response_bytes", "response_sizes"} + vals := func(a monitor.Agg) []any { + return []any{a.Requests, a.Errors, a.ClientErrors, int64(0), a.DurationSum, a.DurationMin, a.DurationMax, a.Durations.Encode(), a.RequestBytes, a.ResponseBytes, a.Sizes.Encode()} + } + written := 0 + var days, hours [][]any + var marks [][]any + for d := today.AddDate(0, 0, 1-loadDays); d.Before(today); d = d.AddDate(0, 0, 1) { + for route := range loadRoutes { + for si, status := range statuses { + a := agg(route, int64(perKeyDay*shares[si]), status) + days = append(days, append([]any{ws, prop, d, env, int32(0), ids[route], monitor.Method("GET"), status, int32(0), int16(monitor.SourceServer), int16(0)}, vals(a)...)) + } + } + for hr := range 24 { + marks = append(marks, []any{ws, prop, d.Add(time.Duration(hr) * time.Hour), d, now, false, false, now}) + } + } + for hr := today; hr.Add(time.Hour).Before(now.Truncate(time.Hour).Add(time.Second)); hr = hr.Add(time.Hour) { + for route := range loadRoutes { + for si, status := range statuses { + a := agg(route, int64(perKeyDay*shares[si]/24), status) + hours = append(hours, append([]any{ws, prop, hr, env, int32(0), ids[route], monitor.Method("GET"), status, int32(0), int16(monitor.SourceServer), int16(0)}, vals(a)...)) + } + } + marks = append(marks, []any{ws, prop, hr, today, now, false, false, nil}) + } + for _, c := range []struct { + table string + cols []string + rows [][]any + }{ + {"monitor_day", cols, days}, + {"monitor_hour", append([]string{"workspace_id", "property_id", "hour"}, cols[3:]...), hours}, + {"monitor_rollup_hours", []string{"workspace_id", "property_id", "hour", "day", "built_at", "dirty", "capped", "day_built_at"}, marks}, + } { + n, err := h.st.Pool.CopyFrom(ctx, pgx.Identifier{c.table}, c.cols, pgx.CopyFromRows(c.rows)) + if err != nil { + t.Fatalf("%s: %v", c.table, err) + } + written += int(n) + } + if _, err := h.st.Pool.Exec(ctx, "ANALYZE monitor_day, monitor_hour, monitor_rollup_hours"); err != nil { + t.Fatal(err) + } + return written +} + +// loadMonitorIngest sends bursts of records for the configured time while a goroutine flushes every +// ten seconds, then waits until the last records are visible to a query. +func loadMonitorIngest(t *testing.T, h *harness, rep *loadReport, key, id string) { + rate := envInt("YUVA_TEST_MONITOR_RATE", 2000) + seconds := envInt("YUVA_TEST_MONITOR_SECONDS", 30) + batch := 100 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var flushes []time.Duration + var flushMu sync.Mutex + flushDone := make(chan struct{}) + go func() { + defer close(flushDone) + tick := time.NewTicker(monitor.FlushInterval) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + } + s := time.Now() + if err := h.srv.FlushMonitor(context.Background()); err != nil { + t.Errorf("flush: %v", err) + return + } + flushMu.Lock() + flushes = append(flushes, time.Since(s)) + flushMu.Unlock() + } + }() + client := &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 64, MaxConnsPerHost: 64}} + url := h.url + "/v1/analytics/properties/" + id + "/requests" + var latMu sync.Mutex + var latencies []time.Duration + var accepted, refused atomic.Int64 + mem := sampleMemory() + cpu0, began := cpuSeconds(), time.Now() + jobs := make(chan int, rate) + var wg sync.WaitGroup + at := h.clock.Now().Format(time.RFC3339Nano) + for w := range 16 { + wg.Add(1) + go func() { + defer wg.Done() + r := rand.New(rand.NewPCG(uint64(w), 9)) + for range jobs { + recs := make([]map[string]any, batch) + for i := range recs { + route := r.IntN(loadRoutes) + status := 200 + switch x := r.IntN(100); { + case x < 3: + status = 404 + case x < 4: + status = 500 + } + recs[i] = map[string]any{"method": "GET", "route": loadRoute(route), "status": status, "duration_us": loadDuration(r, route), "time": at, + "consumer": fmt.Sprintf("client-%d", r.IntN(loadConsumers)), "response_size": 500 + r.IntN(2000)} + } + body, _ := json.Marshal(map[string]any{"records": recs}) + req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+key) + req.Header.Set("Content-Type", "application/json") + s := time.Now() + res, err := client.Do(req) + took := time.Since(s) + if err != nil { + refused.Add(int64(batch)) + continue + } + res.Body.Close() + latMu.Lock() + latencies = append(latencies, took) + latMu.Unlock() + if res.StatusCode == http.StatusAccepted { + accepted.Add(int64(batch)) + } else { + refused.Add(int64(batch)) + } + } + }() + } + per := rate / batch / 10 + tick := time.NewTicker(100 * time.Millisecond) + for time.Since(began) < time.Duration(seconds)*time.Second { + <-tick.C + for range max(per, 1) { + jobs <- 1 + } + } + tick.Stop() + close(jobs) + wg.Wait() + lastSent := time.Now() + sent := lastSent.Sub(began) + want := accepted.Load() + q := map[string]any{"dataset": "api", "metrics": []string{"requests"}, "range": map[string]any{"period": "15m"}} + var visible time.Duration + owner := h.client() + owner.bearer = key + frozen := h.clock.Now() + for { + h.clock.Set(frozen.Add(time.Since(lastSent))) + res := owner.do(http.MethodPost, "/v1/analytics/properties/"+id+"/query", q) + if res.status == http.StatusOK && int64(totals(res)["requests"].(float64)) >= want { + visible = time.Since(lastSent) + break + } + if time.Since(lastSent) > time.Minute { + t.Fatalf("records sent were not visible within a minute: %s", res.raw) + } + time.Sleep(250 * time.Millisecond) + } + cancel() + <-flushDone + cpu := cpuSeconds() - cpu0 + total, heap := mem.end() + flushMu.Lock() + defer flushMu.Unlock() + rep.add(t, "## Ingest: %d records/s in batches of %d for %ds, 500 routes, 5,000 consumers", rate, batch, seconds) + rep.add(t, "- accepted %d (%.0f/s), refused %d; CPU %.1f cores (server and load generator in one process)", want, float64(want)/sent.Seconds(), refused.Load(), cpu/sent.Seconds()) + rep.add(t, "- request p50 %v, p99 %v (target p99 < 100ms); visible to a query %v after the last record (target < 15s)", + quantile(latencies, 0.5).Round(time.Microsecond), quantile(latencies, 0.99).Round(time.Microsecond), visible.Round(time.Millisecond)) + rep.add(t, "- flush p50 %v, max %v over %d flushes; peak Go memory %d MiB (heap objects %d MiB)", quantile(flushes, 0.5).Round(time.Millisecond), + slices.Max(append(flushes, 0)).Round(time.Millisecond), len(flushes), total>>20, heap>>20) + if p99 := quantile(latencies, 0.99); p99 > 100*time.Millisecond { + t.Errorf("ingest p99 %v over 100ms", p99) + } + if visible > 15*time.Second { + t.Errorf("records visible after %v, over 15s", visible) + } + if refused.Load() > 0 { + t.Errorf("%d records refused", refused.Load()) + } +} From 3318cf0682846f5893861bd98164d23a68a27cf9 Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:42:20 +0200 Subject: [PATCH 06/69] docs: API monitoring architecture follows what M16.1 shipped Route inference also applies to host-sent patterns, the id rule for long tokens, path_rules as route rules, status class none, 422 property_kind, local minute labels, the 5-second cache on open ranges, the late counter and the monthly limit counting received records. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index efed59fb..8db65fb4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1716,8 +1716,9 @@ request. Decisions: M15 table that hangs off a property works for an API without a parallel table. Its `domains` (0–50) are the API's hostnames, its `platforms` the apps allowed to send app-side records (`ios`, `android`, or none); it takes no goals, props, funnels or consent features, and `audience` does not - apply. New columns: `apdex_ms` (500) and `monitor_hour_days` (90). The collection endpoint and the - site views refuse an API property; the API endpoints refuse a site. + apply. New columns: `apdex_ms` (500) and `monitor_hour_days` (90). An API property is created with + `domains: []`. The collection endpoint answers `404` for an API property's key; site views and API + endpoints called on the other kind answer `422 property_kind`. - **A record.** Method (the RFC 9110 methods and `PATCH`, else `_OTHER`), route, status (100–599, or 0 when no response was sent), duration (microseconds), request and response size, time, and optionally `consumer` (1–100 characters, chosen by the host: an API key's name or a client id, never a person's @@ -1725,14 +1726,16 @@ request. Decisions: `production` by default), `service` (≤ 100), `release` (≤ 80, only for deploy markers), `ip` (only to find the country) and `sample_rate` (1–10,000, 1 by default). No body, header, query string, fragment, user agent or address is accepted or kept. 5xx and 0 are errors, 4xx client errors; both - rates are shown. + rates are shown. Status 0 has the status class `none`. - **Routes, never raw paths.** A record names its route pattern (`/users/{id}`: `r.Pattern` from `http.ServeMux`, the framework's router, OpenTelemetry `http.route`). When it can only give `path`, - ingest infers a pattern: segments that are numbers, UUIDs, dates, hex or base64url of 16 or more - characters, e-mail-shaped or longer than 40 characters become `{id}`, and `?`, `#` and what follows - are cut first. A record without a pattern that answered 404 or 405 goes to `(unmatched)`, so scanners + ingest infers a pattern: segments that are numbers, UUIDs, dates, hex, tokens of 16 or more + characters with two or more digits or with both cases and a digit (so `user-preferences` stays), + e-mail-shaped or longer than 40 characters become `{id}`, and `?`, `#` and what follows are cut + first. Patterns the host sends go through the same segment rule, so a raw id never becomes a route + either way. A record without a pattern that answered 404 or 405 goes to `(unmatched)`, so scanners never create routes. `monitor_routes` marks inferred patterns; members hide a route or map inferred - ones onto a pattern of their own (route rules, applied at ingest from then on). Caps per property: + ones onto a pattern of their own (the property's `path_rules`, applied at ingest from then on). Caps per property: 1,000 routes, 10,000 consumers, 50 API versions, 200 app versions, 20 environments; 100 services per workspace. A value past its cap is counted as `(other)` and the panel says which cap was reached. - **Percentiles from mergeable sketches.** Durations go into a sparse log-bucketed histogram with base @@ -1764,14 +1767,17 @@ request. Decisions: half-hour offsets included; a time zone change applies from the next local day. The `monitor_rollup` job on the `analytics` queue folds a closed hour's minute rows into one row per key, compacts its dimension rows to one writer, and folds finished local days, tracked in `monitor_rollup_hours` - (built, dirty) like `analytics_rollup_days`. Queries read folded rows for closed hours and minute and + (built, dirty, and `day` with `day_built_at` for the day fold) like `analytics_rollup_days`. A + process that could not write an hour's dimension rows within 5 minutes after the hour ended drops + them and logs it, so they are never counted twice. Queries read folded rows for closed hours and minute and writer rows for open ones. An hour holds at most 50,000 dimension rows per property; later new combinations of that hour count as `(other)` and the hour is marked capped. Dictionaries give the rollups integer ids: `monitor_routes` (pattern, inferred, hidden, `apdex_ms`, first and last seen) and `monitor_names` (consumers, API versions, app versions, environments, countries per property). - **Sampling.** Off by default: records are aggregated on arrival, so cost grows with distinct keys, not with requests. An SDK may sample successful requests (`sampleRate`), never 4xx, 5xx or failed ones; - a record carries its weight, counters and buckets add it, and `sampled` keeps the weighted requests + a record carries its weight (one with `sample_rate` above 1 and an error status is refused as + invalid), counters and buckets add it, and `sampled` keeps the weighted requests of each row, so views, exports and the Prometheus endpoint mark a range that holds any as estimated. An OTLP span weighs the adjusted count of its tracestate `ot=th` threshold (OpenTelemetry's probability sampling, still a Development spec) and 1 without one, so a head-sampled exporter @@ -1799,7 +1805,8 @@ request. Decisions: the first function that sees it, then dropped. Limits per property (`YUVA_MONITOR_RECORDS_PER_MINUTE`, 600,000) and per key; in-memory keys per process are capped (`YUVA_MONITOR_BUFFER_KEYS`, 200,000; full answers `503`). `monitor_ingest_stats` counts accepted, - invalid, rate, over_limit and buffer per property and day. + invalid, late, rate, over_limit and buffer per property and day. The monthly limit counts records + received, not their sampled weight; records past it are counted as `over_limit`. - **SDKs (M16.2).** `sdk/go/analytics` gets `Middleware` for `net/http`: it wraps the mux and reads `r.Pattern` after the handler ran (`ServeMux` sets it on the same request), without its method and host, or a `Route func(*http.Request) string`; batches of 500 or every 5 seconds through the existing @@ -1815,12 +1822,14 @@ request. Decisions: `p90`, `p95`, `p99`, `avg_duration`, `max_duration`, `apdex`, `request_bytes`, `response_bytes`, `response_size_p50`, `response_size_p95` and `conversations`; dimensions route, method, status, status class, API version, environment, service, source, platform, consumer, country, app version and - `time:minute` (inside the minute tier, at most 1,440 buckets), `time:hour`, `time:day`, `time:week`, + `time:minute` (inside the minute tier, at most 1,440 buckets, labelled in the property's local time), `time:hour`, `time:day`, `time:week`, `time:month`. `range` also takes `15m`, `1h`, `6h` and `24h` and minute-aligned `since` and `until`. Any combination of core dimensions filters and breaks down from every tier; at most one extra dimension (consumer, country or app version) joins them, from hours and days (`422 dimension_combination` otherwise). Comparisons, the 4 query slots per workspace and the cache are - M15's. New routes are first seen in the range, silent routes were called in the previous period and + M15's, except that answers over open ranges are cached for 5 seconds instead of 30, so new records + show within 15 seconds. Results carry `meta.estimated` and `meta.caps_reached`, and the usage object + `requests` and `requests_limit`. New routes are first seen in the range, silent routes were called in the previous period and not since. A new `release` per service and environment writes a `deploy` annotation, which gains a time (`at`) so minute charts place it; Yuva's own annotations stay read-only. - **Uptime checks (M16.4).** `monitor_checks`: name, `http` or `https` URL without credentials, `GET` From b99fb76cb5829f6bf698311a02e88edc08005104 Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:45:33 +0200 Subject: [PATCH 07/69] docs: API monitoring guide and settings in en, tr and de Refs #64 Co-Authored-By: Claude Opus 5.5 --- docs/analytics-api.md | 198 +++++++++++++++++++++++++++++++++++++ docs/analytics.md | 3 +- docs/configuration.md | 18 ++++ docs/de/analytics-api.md | 206 +++++++++++++++++++++++++++++++++++++++ docs/de/analytics.md | 3 +- docs/de/configuration.md | 18 ++++ docs/de/headless.md | 2 +- docs/de/operations.md | 1 + docs/headless.md | 2 +- docs/operations.md | 1 + docs/tr/analytics-api.md | 200 +++++++++++++++++++++++++++++++++++++ docs/tr/analytics.md | 3 +- docs/tr/configuration.md | 18 ++++ docs/tr/headless.md | 2 +- docs/tr/operations.md | 1 + site/src/lib/docs.ts | 1 + 16 files changed, 671 insertions(+), 6 deletions(-) create mode 100644 docs/analytics-api.md create mode 100644 docs/de/analytics-api.md create mode 100644 docs/tr/analytics-api.md diff --git a/docs/analytics-api.md b/docs/analytics-api.md new file mode 100644 index 00000000..38f4918a --- /dev/null +++ b/docs/analytics-api.md @@ -0,0 +1,198 @@ +# API monitoring + +An API property counts the requests your own API serves: how many, how many failed, and how long they took, by +route. Your server reports each request it answered, and Yuva adds them up per minute and hour, without +keeping a row per request, a header, a body or an IP address. This builds on [Analytics](analytics.md). + +## Make an API property + +An API property is an analytics property with `kind: api`. It has no domains and no tracking script. Owners and +admins create it with a key that has the scope `analytics:manage`: + +```sh +curl https://support.example.com/v1/analytics/properties \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "kind": "api", "name": "Shop API", "domains": [], "timezone": "Europe/Berlin" }' +``` + +The answer carries the property's `id`. The property's time zone decides which hours and days its numbers are +counted in, and the minutes of a `time:minute` query are in it too. + +Then make the key that sends the records, in **Settings → API keys**, with the scope `analytics:write`, and keep it +on the server. A member's session is refused (`403 forbidden`), and a site property answers +`422 property_kind`. A server started with `YUVA_MONITOR=off` has no API properties. + +## Send requests from Go + +```sh +go get github.com/productdevbook/yuva/sdk/go/analytics +``` + +`Middleware` wraps a `net/http` handler and reports every request it answers. Wrap the `http.ServeMux` itself, so +the pattern the mux matched is known: `GET /users/{id}` is recorded as the route `/users/{id}`. + +```go +import "github.com/productdevbook/yuva/sdk/go/analytics" + +mux := http.NewServeMux() +mux.HandleFunc("GET /users/{id}", getUser) + +monitor, err := analytics.New(analytics.Config{ + BaseURL: "https://support.example.com", + APIKey: os.Getenv("YUVA_API_KEY"), + PropertyID: os.Getenv("YUVA_API_PROPERTY"), +}) +if err != nil { + log.Fatal(err) +} +defer monitor.Close(context.Background()) + +handler := monitor.Middleware(analytics.Monitor{ + Service: "shop-api", + Consumer: func(r *http.Request) string { return apiKeyName(r) }, +})(mux) +log.Fatal(http.ListenAndServe(":8080", handler)) +``` + +Records go out in gzip batches of up to 500 every 5 seconds, with the retries of [server-side events](analytics-server.md#go). +Call `Close` before the process exits, or the records still waiting are lost. + +| `Monitor` | Default | | +|---|---|---| +| `Route` | the mux's pattern | Names the route of a request when your router is not a `ServeMux`. | +| `Consumer` | none | Who called: an API key's name or a client id. Never a person's address; a value shaped like an e-mail address is stored as `(scrubbed)`. | +| `APIVersion` | none | The API version the request used. | +| `Environment` | `production` | Tells apart `production`, `staging` and the like. | +| `Service` | none | The service that served the request. | +| `Release` | none | The service's release. It is accepted but not shown yet. | +| `SampleRate` | every request | `10` sends one in ten successful requests (status 1 to 399), each standing for ten. Errors and client errors are always sent. | +| `Country` | off | Sends the caller's address (`Config.ClientIP`) so Yuva can count the request's country. It is shortened on arrival and never stored. | + +A request no pattern matched is sent with its path. A handler that panics is recorded with the status it wrote, +or `0` when it wrote none. + +## Send requests with curl + +Any other stack posts the same records. `POST /v1/analytics/properties/{propertyId}/requests` takes 1 to 1,000 +records and answers `202`. The body may be gzip-compressed with `Content-Encoding: gzip`. + +```sh +curl https://support.example.com/v1/analytics/properties/PROPERTY_ID/requests \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "records": [{ + "method": "GET", + "route": "/users/{id}", + "status": 200, + "duration_us": 18400, + "response_size": 612, + "consumer": "mobile-app", + "environment": "production" + }] + }' +``` + +```json +{ "accepted": 1, "errors": [] } +``` + +| Field | | +|---|---| +| `method` | Required, 1 to 20 characters. The RFC 9110 methods and `PATCH` are kept; anything else counts as `_OTHER`. | +| `route` | The pattern the router matched, starting with `/`, up to 300 characters, without method or host. Give this or `path`. | +| `path` | The request path, for a router that knows no pattern. Only the pattern worked out from it is kept. | +| `status` | Required, 100 to 599, or `0` when no response was sent. | +| `duration_us` | Required, the duration in microseconds, up to 24 hours. | +| `request_size`, `response_size` | Bytes of the bodies. | +| `time` | When the request started, RFC 3339. Now when left out. | +| `consumer` | Up to 100 characters; see above. | +| `api_version`, `environment`, `service`, `release` | Up to 40, 40, 100 and 80 characters. `environment` is `production` when left out. | +| `ip` | The caller's address, only to find its country. | +| `sample_rate` | 1 to 10,000: this record stands for that many requests. Only a status of 1 to 399 may be sampled; a record with `sample_rate` above 1 and an error status is refused. | + +## Routes + +A route names a part of your API, never one record, so an id never becomes a route of its own. + +- **Routes you send are cleaned.** In a `route` or a `path`, every segment that looks like an id becomes `{id}`: + numbers, UUIDs, dates, hex strings of 16 characters or more, other tokens of 16 or more letters, digits, `-` and + `_` with at least two digits (or both cases and a digit), e-mail-shaped values, and anything over 40 + characters. Segments that start like a router's placeholder (`{id}`, `:id`, `*`, `[id]`, ``) stay as they are. + So `/users/42` and `/users/9f8e7d6c5b4a3210` both count as `/users/{id}`. +- **A 404 or 405 without a pattern is `(unmatched)`.** A record that gives only a `path` and was answered with + `404` or `405` is counted under the route `(unmatched)`, so scanners probing `/wp-login.php` never make + routes. +- **Route rules.** The property's `path_rules` map a route or path onto a route of your own, before the cleaning + above. A rule is `{"pattern": "/legacy/**", "replacement": "/legacy"}`: `*` matches one segment and `**` any + number of them, the first matching rule wins, and the replacement is used as it is. Set them with + `PATCH /v1/analytics/properties/{propertyId}`. +- **Caps.** A property counts at most 1,000 routes, 10,000 consumers, 50 API versions and 20 environments, and a + workspace 100 services. Past a cap, new values are counted as `(other)`, and the query answer lists the cap + in `meta.caps_reached`. + +## What is recorded + +Recorded, per minute, hour and day: the number of requests, those with status 500 to 599 or `0` (errors) and +with status 400 to 499 (client errors), and the distribution of durations that gives p50, p95 and p99, by route, +method, status and environment. Service, API version and consumer are kept with them. + +Never recorded, and not accepted by the API: request or response bodies, headers, query strings, fragments +and user agents. Postgres holds no row per request. An `ip` is shortened (IPv4 `/24`, IPv6 `/48`), looked up for +the country and dropped in the same call; no address is stored. The Go middleware sends an address only with +`Country: true`. + +## Read the numbers + +In the panel, the property's **Overview** shows requests, error rates and latency over time, and **Routes** the +routes with their traffic, errors and latency. + +The same numbers come from the analytics query, `POST /v1/analytics/properties/{propertyId}/query`, with +`"dataset": "api"` (scope `analytics:read`): + +```sh +curl https://support.example.com/v1/analytics/properties/PROPERTY_ID/query \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "dataset": "api", + "metrics": ["requests", "error_rate", "p95"], + "dimensions": ["route"], + "range": { "period": "24h" }, + "limit": 20 + }' +``` + +| | | +|---|---| +| Metrics | `requests`, `requests_per_minute`, `errors`, `error_rate`, `client_errors`, `client_error_rate`, `p50`, `p95`, `p99`, `avg_duration`. Rates are percent, durations milliseconds. The percentiles are accurate to 1.1%. | +| Dimensions and filters | `route`, `method`, `status`, `status_class` (`2xx`, `3xx`, `4xx`, `5xx`, or `none` for status `0`), `environment`; at most two dimensions, one of them may be a time grain. | +| Time grains | `time:minute`, `time:hour`, `time:day`. Minutes and hours are in the property's local time. | +| Range | `15m`, `1h`, `6h`, `24h` up to the current minute, or `today`, `7d`, `30d` and the other periods, or `from` and `to`. | + +`time:minute` reaches back [`YUVA_MONITOR_MINUTE_DAYS`](configuration.md#api-monitoring) days (8 by default) and +returns at most 1,440 minutes; a longer or older range is refused with `422 range_too_large`. Hourly numbers are +kept for the property's `monitor_hour_days` (90 by default, 7 to 400), daily numbers as long as the property's +`retention_months`. An answer that rests on records sent with `sample_rate` above 1 has `meta.estimated: true`. +`dataset: api` on a site property, and a query without it on an API property, answer `422 property_kind`. An API +query takes no `goal_id`, `prop`, `segment_id`, `compare` or +`insights`. + +A record shows up in a query within about 15 seconds. + +## What comes back + +The answer is `202` when the request was valid, whatever happened to the records. + +- **`errors`** lists the records left out as invalid, each with its `index` in `records` and a `detail`. The others are taken. +- **Taken but not counted:** records older than 72 hours; records over `YUVA_MONITOR_RECORDS_PER_MINUTE` for the + property or for the key; records over the workspace's monthly limit. +- **The whole request is refused** for a body that breaks the shape above (`400`), a missing or wrong key (`401`), + a key without `analytics:write` or a member's session (`403`), an unknown property (`404`), a site property + (`422`) and an encoding other than gzip (`415`). A server that holds as many open rows as it may answers `503` + with `Retry-After`: send the batch again after that. + +The monthly limit counts the records received, not the requests they stand for: a sampled record is one record +whatever its `sample_rate`. `GET /v1/analytics/properties` shows the month's `requests` and the `requests_limit` +under `usage`. diff --git a/docs/analytics.md b/docs/analytics.md index f2e667b6..c8fdfd8f 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -269,7 +269,8 @@ A website linked to an inbox also shows the support its pages lead to, with insi [Support and traffic](analytics-support.md). The same property counts your iOS and Android apps ([Analytics in apps](analytics-apps.md)) and events your backend -sends ([Server-side events](analytics-server.md)). +sends ([Server-side events](analytics-server.md)). A property of kind `api` counts the requests of your own API +instead: [API monitoring](analytics-api.md). **Channels** sort where a visit came from: Direct, Organic search, Paid search, Social, Paid social, E-mail, Referral, **AI assistants** (visits from ChatGPT and other assistants) and Internal. diff --git a/docs/configuration.md b/docs/configuration.md index 0d05a624..e946b017 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -166,6 +166,24 @@ The commands, run as in [Operator commands](operations.md#operator-commands): | `analytics geoip-update` | Downloads this month's DB-IP Lite database into `YUVA_GEOIP_DIR` unless it is there, and prints the credit to give. With `file`, checks your file. | | `workspace analytics-limit --workspace ` | Sets the workspace's events per month; `0` removes the limit. | +## API monitoring + +[Watching your API](analytics-api.md). Part of analytics: `YUVA_ANALYTICS=off` turns it off too. + +| Variable | Default | Meaning | +|---|---|---| +| `YUVA_MONITOR` | `on` | `off` removes API properties and their request records and queries. | +| `YUVA_MONITOR_MONTHLY_REQUESTS` | `10000000` when `YUVA_SIGNUP=open`, else `0` | Request records one workspace may send per UTC month; more are not counted until the month ends. `0` is no limit. Per workspace, `yuva workspace monitor-limit` overrides it. | +| `YUVA_MONITOR_RECORDS_PER_MINUTE` | `600000` | Request records one API property, and one API key, may send per minute; more are not counted. At least 1. | +| `YUVA_MONITOR_BUFFER_KEYS` | `200000` | Open rows one process holds before writing them (every 10 seconds). With that many open, the endpoint answers `503` with `Retry-After`. At least 1000. | +| `YUVA_MONITOR_MINUTE_DAYS` | `8` | Days the per-minute numbers are kept, from 2 to 31; `time:minute` queries reach back that far. | + +The command, run as in [Operator commands](operations.md#operator-commands): + +| Command | What it does | +|---|---| +| `workspace monitor-limit --workspace ` | Sets the workspace's request records per month; `0` removes the limit. | + ## Chat | Variable | Default | Meaning | diff --git a/docs/de/analytics-api.md b/docs/de/analytics-api.md new file mode 100644 index 00000000..d87a3756 --- /dev/null +++ b/docs/de/analytics-api.md @@ -0,0 +1,206 @@ +# API-Monitoring + +Eine API-Property zählt die Requests, die Ihre eigene API beantwortet: wie viele es waren, wie viele fehlschlugen +und wie lange sie dauerten, nach Route. Ihr Server meldet jeden Request, den er beantwortet hat; Yuva addiert sie +pro Minute und Stunde, ohne eine Zeile pro Request, ohne Header, Body oder IP-Adresse zu speichern. Diese Seite +baut auf [Analytics](analytics.md) auf. + +## Eine API-Property anlegen + +Eine API-Property ist eine Analytics-Property mit `kind: api`. Sie hat keine Domains und kein Tracking-Skript. +Owner und Admins legen sie mit einem Schlüssel mit dem Scope `analytics:manage` an: + +```sh +curl https://support.example.com/v1/analytics/properties \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "kind": "api", "name": "Shop API", "domains": [], "timezone": "Europe/Berlin" }' +``` + +Die Antwort enthält die `id` der Property. Die Zeitzone der Property bestimmt, nach welchen Stunden und Tagen ihre +Zahlen gezählt werden; auch die Minuten einer `time:minute`-Abfrage liegen in ihr. + +Legen Sie dann unter **Settings → API keys** den Schlüssel an, der die Records sendet, mit dem Scope +`analytics:write`, und behalten Sie ihn auf dem Server. Die Sitzung eines Mitglieds wird abgelehnt +(`403 forbidden`), eine Website-Property antwortet mit `422 property_kind`. Ein mit `YUVA_MONITOR=off` gestarteter +Server hat keine API-Properties. + +## Requests aus Go senden + +```sh +go get github.com/productdevbook/yuva/sdk/go/analytics +``` + +`Middleware` umschließt einen `net/http`-Handler und meldet jeden Request, den er beantwortet. Umschließen Sie den +`http.ServeMux` selbst, damit das vom Mux gefundene Muster bekannt ist: `GET /users/{id}` wird als Route +`/users/{id}` erfasst. + +```go +import "github.com/productdevbook/yuva/sdk/go/analytics" + +mux := http.NewServeMux() +mux.HandleFunc("GET /users/{id}", getUser) + +monitor, err := analytics.New(analytics.Config{ + BaseURL: "https://support.example.com", + APIKey: os.Getenv("YUVA_API_KEY"), + PropertyID: os.Getenv("YUVA_API_PROPERTY"), +}) +if err != nil { + log.Fatal(err) +} +defer monitor.Close(context.Background()) + +handler := monitor.Middleware(analytics.Monitor{ + Service: "shop-api", + Consumer: func(r *http.Request) string { return apiKeyName(r) }, +})(mux) +log.Fatal(http.ListenAndServe(":8080", handler)) +``` + +Records gehen alle 5 Sekunden in gzip-Batches von bis zu 500 hinaus, mit den Wiederholungen der +[serverseitigen Events](analytics-server.md#go). Rufen Sie `Close` auf, bevor der Prozess endet, sonst gehen die +wartenden Records verloren. + +| `Monitor` | Standard | | +|---|---|---| +| `Route` | das Muster des Mux | Nennt die Route eines Requests, wenn Ihr Router kein `ServeMux` ist. | +| `Consumer` | keiner | Wer aufgerufen hat: der Name eines API-Schlüssels oder eine Client-ID. Nie die Adresse einer Person; ein Wert in E-Mail-Form wird als `(scrubbed)` gespeichert. | +| `APIVersion` | keine | Die API-Version, die der Request nutzte. | +| `Environment` | `production` | Unterscheidet `production`, `staging` und dergleichen. | +| `Service` | keiner | Der Service, der den Request beantwortet hat. | +| `Release` | keines | Das Release des Service. Es wird angenommen, aber noch nicht angezeigt. | +| `SampleRate` | jeder Request | `10` sendet jeden zehnten erfolgreichen Request (Status 1 bis 399), jeder steht für zehn. Fehler und Client-Fehler werden immer gesendet. | +| `Country` | aus | Sendet die Adresse des Aufrufers (`Config.ClientIP`), damit Yuva das Land des Requests zählen kann. Sie wird beim Eingang gekürzt und nie gespeichert. | + +Ein Request, auf den kein Muster passte, wird mit seinem Pfad gesendet. Ein Handler, der in Panik gerät, wird mit +dem Status erfasst, den er geschrieben hat, oder mit `0`, wenn er keinen schrieb. + +## Requests mit curl senden + +Jeder andere Stack sendet dieselben Records. `POST /v1/analytics/properties/{propertyId}/requests` nimmt 1 bis +1.000 Records und antwortet mit `202`. Der Body darf mit `Content-Encoding: gzip` komprimiert sein. + +```sh +curl https://support.example.com/v1/analytics/properties/PROPERTY_ID/requests \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "records": [{ + "method": "GET", + "route": "/users/{id}", + "status": 200, + "duration_us": 18400, + "response_size": 612, + "consumer": "mobile-app", + "environment": "production" + }] + }' +``` + +```json +{ "accepted": 1, "errors": [] } +``` + +| Feld | | +|---|---| +| `method` | Pflicht, 1 bis 20 Zeichen. Die Methoden aus RFC 9110 und `PATCH` bleiben erhalten; alles andere zählt als `_OTHER`. | +| `route` | Das Muster, das der Router gefunden hat, beginnt mit `/`, bis zu 300 Zeichen, ohne Methode und Host. Geben Sie dies oder `path` an. | +| `path` | Der Pfad des Requests, für einen Router, der kein Muster kennt. Nur das daraus ermittelte Muster wird behalten. | +| `status` | Pflicht, 100 bis 599, oder `0`, wenn keine Antwort gesendet wurde. | +| `duration_us` | Pflicht, die Dauer in Mikrosekunden, bis zu 24 Stunden. | +| `request_size`, `response_size` | Bytes der Bodys. | +| `time` | Wann der Request begann, RFC 3339. Jetzt, wenn leer. | +| `consumer` | Bis zu 100 Zeichen; siehe oben. | +| `api_version`, `environment`, `service`, `release` | Bis zu 40, 40, 100 und 80 Zeichen. `environment` ist `production`, wenn leer. | +| `ip` | Die Adresse des Aufrufers, nur um sein Land zu finden. | +| `sample_rate` | 1 bis 10.000: Dieser Record steht für so viele Requests. Nur ein Status von 1 bis 399 darf gesampelt werden; ein Record mit `sample_rate` über 1 und einem Fehlerstatus wird abgelehnt. | + +## Routen + +Eine Route benennt einen Teil Ihrer API, nie einen einzelnen Datensatz; eine ID wird also nie zu einer eigenen Route. + +- **Gesendete Routen werden bereinigt.** In einer `route` oder einem `path` wird jedes Segment, das wie eine ID + aussieht, zu `{id}`: Zahlen, UUIDs, Datumsangaben, Hex-Strings ab 16 Zeichen, andere Token ab 16 Buchstaben, + Ziffern, `-` und `_` mit mindestens zwei Ziffern (oder Groß- und Kleinbuchstaben und einer Ziffer), + Werte in E-Mail-Form und alles über 40 Zeichen. Segmente, die wie ein Platzhalter des Routers beginnen (`{id}`, + `:id`, `*`, `[id]`, ``), bleiben unverändert. So zählen `/users/42` und `/users/9f8e7d6c5b4a3210` beide als + `/users/{id}`. +- **Ein 404 oder 405 ohne Muster ist `(unmatched)`.** Ein Record, der nur einen `path` angibt und mit `404` oder + `405` beantwortet wurde, zählt unter der Route `(unmatched)`, damit Scanner, die `/wp-login.php` anfragen, keine + Routen erzeugen. +- **Routenregeln.** Die `path_rules` der Property bilden vor der obigen Bereinigung eine Route oder einen Pfad auf + eine Route Ihrer Wahl ab. Eine Regel ist `{"pattern": "/legacy/**", "replacement": "/legacy"}`: `*` passt auf + ein Segment und `**` auf beliebig viele, die erste passende Regel gewinnt, und die Ersetzung wird unverändert + verwendet. Setzen Sie sie mit `PATCH /v1/analytics/properties/{propertyId}`. +- **Obergrenzen.** Eine Property zählt höchstens 1.000 Routen, 10.000 Consumer, 50 API-Versionen und 20 + Environments, ein Workspace 100 Services. Über einer Grenze zählen neue Werte als `(other)`, und die Antwort der + Abfrage nennt die Grenze in `meta.caps_reached`. + +## Was erfasst wird + +Erfasst wird pro Minute, Stunde und Tag: die Zahl der Requests, die mit Status 500 bis 599 oder `0` (Fehler) und +mit Status 400 bis 499 (Client-Fehler) sowie die Verteilung der Dauern, aus der p50, p95 und p99 entstehen, nach +Route, Methode, Status und Environment. Service, API-Version und Consumer werden dazu behalten. + +Nie erfasst und von der API nicht angenommen: Request- oder Response-Bodys, Header, Query-Strings, Fragmente und +User-Agents. Postgres hält keine Zeile pro Request. Eine `ip` wird gekürzt (IPv4 `/24`, IPv6 `/48`), für das Land +nachgeschlagen und im selben Aufruf verworfen; keine Adresse wird gespeichert. Die Go-Middleware sendet eine Adresse +nur mit `Country: true`. + +## Die Zahlen lesen + +Im Panel zeigt die **Übersicht** der Property Requests, Fehlerraten und Latenz im Zeitverlauf, **Routen** die Routen +mit Traffic, Fehlern und Latenz. + +Dieselben Zahlen kommen aus der Analytics-Abfrage, `POST /v1/analytics/properties/{propertyId}/query`, mit +`"dataset": "api"` (Scope `analytics:read`): + +```sh +curl https://support.example.com/v1/analytics/properties/PROPERTY_ID/query \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "dataset": "api", + "metrics": ["requests", "error_rate", "p95"], + "dimensions": ["route"], + "range": { "period": "24h" }, + "limit": 20 + }' +``` + +| | | +|---|---| +| Metriken | `requests`, `requests_per_minute`, `errors`, `error_rate`, `client_errors`, `client_error_rate`, `p50`, `p95`, `p99`, `avg_duration`. Raten sind Prozent, Dauern Millisekunden. Die Perzentile sind auf 1,1 % genau. | +| Dimensionen und Filter | `route`, `method`, `status`, `status_class` (`2xx`, `3xx`, `4xx`, `5xx`, oder `none` für Status `0`), `environment`; höchstens zwei Dimensionen, eine davon darf eine Zeitauflösung sein. | +| Zeitauflösungen | `time:minute`, `time:hour`, `time:day`. Minuten und Stunden gelten in der Ortszeit der Property. | +| Zeitraum | `15m`, `1h`, `6h`, `24h` bis zur aktuellen Minute, oder `today`, `7d`, `30d` und die anderen Zeiträume, oder `from` und `to`. | + +`time:minute` reicht [`YUVA_MONITOR_MINUTE_DAYS`](configuration.md#api-monitoring) Tage zurück (standardmäßig 8) und +liefert höchstens 1.440 Minuten; ein längerer oder älterer Zeitraum wird mit `422 range_too_large` abgelehnt. +Stundenwerte werden für `monitor_hour_days` der Property aufbewahrt (standardmäßig 90, 7 bis 400), Tageswerte so +lange wie ihre `retention_months`. Eine Antwort, die auf mit `sample_rate` über 1 gesendeten Records beruht, hat +`meta.estimated: true`. `dataset: api` bei einer Website-Property und eine Abfrage ohne dieses Feld bei einer +API-Property antworten mit `422 property_kind`. Eine API-Abfrage nimmt kein `goal_id`, `prop`, `segment_id`, +`compare` und kein `insights`. + +Ein Record erscheint innerhalb von etwa 15 Sekunden in einer Abfrage. + +## Was zurückkommt + +Die Antwort ist `202`, wenn die Anfrage gültig war, was auch mit den Records geschah. + +- **`errors`** listet die als ungültig ausgelassenen Records, jeweils mit ihrem `index` in `records` und einem + `detail`. Die übrigen werden angenommen. +- **Angenommen, aber nicht gezählt:** Records, die älter als 72 Stunden sind; Records über + `YUVA_MONITOR_RECORDS_PER_MINUTE` für die Property oder den Schlüssel; Records über der Monatsgrenze des + Workspace. +- **Die ganze Anfrage wird abgelehnt** bei einem Body, der die obige Form verletzt (`400`), einem fehlenden oder + falschen Schlüssel (`401`), einem Schlüssel ohne `analytics:write` oder der Sitzung eines Mitglieds (`403`), + einer unbekannten Property (`404`), einer Website-Property (`422`) und einer anderen Kodierung als gzip (`415`). + Ein Server, der so viele offene Zeilen hält, wie er darf, antwortet mit `503` und `Retry-After`: Senden Sie den + Batch danach erneut. + +Die Monatsgrenze zählt die empfangenen Records, nicht die Requests, für die sie stehen: Ein gesampelter Record ist +ein Record, egal welche `sample_rate` er hat. `GET /v1/analytics/properties` zeigt die `requests` des Monats und +das `requests_limit` unter `usage`. diff --git a/docs/de/analytics.md b/docs/de/analytics.md index 78ee165c..deee5c83 100644 --- a/docs/de/analytics.md +++ b/docs/de/analytics.md @@ -270,7 +270,8 @@ Eine mit einem Posteingang verknüpfte Website zeigt außerdem, wie viel Support Alerts und E-Mail-Berichten: [Support und Traffic](analytics-support.md). Dieselbe Property zählt auch Ihre iOS- und Android-Apps ([Analytics in Apps](analytics-apps.md)) und die Events, -die Ihr Backend sendet ([Serverseitige Events](analytics-server.md)). +die Ihr Backend sendet ([Serverseitige Events](analytics-server.md)). Eine Property der Art `api` zählt stattdessen +die Requests Ihrer eigenen API: [API-Monitoring](analytics-api.md). **Channels** ordnen ein, woher ein Besuch kam: Direct, Organic search, Paid search, Social, Paid social, E-mail, Referral, **AI assistants** (Besuche von ChatGPT und anderen Assistenten) und Internal. diff --git a/docs/de/configuration.md b/docs/de/configuration.md index 1376af1b..fcb77f30 100644 --- a/docs/de/configuration.md +++ b/docs/de/configuration.md @@ -169,6 +169,24 @@ Die Befehle laufen wie unter [Operator-Befehle](operations.md#operator-befehle): | `analytics geoip-update` | Lädt die DB-IP-Lite-Datenbank dieses Monats nach `YUVA_GEOIP_DIR`, sofern sie nicht schon dort liegt, und gibt die zu nennende Quelle aus. Mit `file` prüft er Ihre Datei. | | `workspace analytics-limit --workspace ` | Setzt die Events pro Monat des Workspaces; `0` hebt das Limit auf. | +## API-Monitoring + +[Ihre API im Blick](analytics-api.md). Teil von Analytics: `YUVA_ANALYTICS=off` schaltet es ebenfalls ab. + +| Variable | Standard | Bedeutung | +|---|---|---| +| `YUVA_MONITOR` | `on` | `off` entfernt API-Properties, ihre Request-Records und Abfragen. | +| `YUVA_MONITOR_MONTHLY_REQUESTS` | `10000000` bei `YUVA_SIGNUP=open`, sonst `0` | Request-Records, die ein Workspace pro UTC-Monat senden darf; weitere werden bis zum Monatsende nicht gezählt. `0` ist keine Grenze. Pro Workspace überschreibt `yuva workspace monitor-limit` den Wert. | +| `YUVA_MONITOR_RECORDS_PER_MINUTE` | `600000` | Request-Records, die eine API-Property und ein API-Schlüssel pro Minute senden dürfen; weitere werden nicht gezählt. Mindestens 1. | +| `YUVA_MONITOR_BUFFER_KEYS` | `200000` | Offene Zeilen, die ein Prozess hält, bevor er sie schreibt (alle 10 Sekunden). Sind so viele offen, antwortet der Endpunkt mit `503` und `Retry-After`. Mindestens 1000. | +| `YUVA_MONITOR_MINUTE_DAYS` | `8` | Tage, die die Minutenwerte aufbewahrt werden, von 2 bis 31; `time:minute`-Abfragen reichen so weit zurück. | + +Der Befehl, ausgeführt wie in [Operator-Befehle](operations.md#operator-befehle): + +| Befehl | Was er tut | +|---|---| +| `workspace monitor-limit --workspace ` | Setzt die Request-Records des Workspace pro Monat; `0` entfernt die Grenze. | + ## Chat | Variable | Standard | Bedeutung | diff --git a/docs/de/headless.md b/docs/de/headless.md index 4b681e01..24e9064a 100644 --- a/docs/de/headless.md +++ b/docs/de/headless.md @@ -114,7 +114,7 @@ Ein Aufruf ohne den nötigen Scope antwortet mit `403 insufficient_scope`. | `workspace:manage` | Workspace-Einstellungen und Nutzung | | `migrations:manage` | [Ein Postfach zu Yuva umziehen](migrate.md): erkennen, anlegen, verbinden, scannen, starten, abschließen, importierten Verlauf entfernen. Nie für Agents oder auf Posteingänge beschränkte Keys. | | `analytics:read` | [Analytics](analytics.md#api): Properties, Zahlen, Realtime, Health | -| `analytics:write` | [Serverseitige Events](analytics-server.md) senden (`POST /v1/analytics/properties/{propertyId}/events`). Nur API-Keys, OAuth-Clients nie angeboten. | +| `analytics:write` | [Serverseitige Events](analytics-server.md) senden (`POST /v1/analytics/properties/{propertyId}/events`) und [API-Request-Records](analytics-api.md) (`POST /v1/analytics/properties/{propertyId}/requests`). Nur API-Keys, OAuth-Clients nie angeboten. | | `analytics:manage` | Properties anlegen, ändern und löschen, ihre Schlüssel erneuern, ihre Daten löschen. Nie für Agents oder auf Posteingänge beschränkte Keys. | | `feedback:write` | `POST /v1/feedback` | diff --git a/docs/de/operations.md b/docs/de/operations.md index 4bc0b0d1..3905095e 100644 --- a/docs/de/operations.md +++ b/docs/de/operations.md @@ -87,6 +87,7 @@ Befehle, die eine ID oder ein Secret ausgeben, schreiben nur das auf stdout. | `workspace broadcasts limit --workspace --daily ` | Setzt die Empfänger pro UTC-Tag des Workspaces; `0` hebt das Limit auf. | | `workspace broadcasts show --workspace ` | Gibt Status, Limit, 30-Tage-Raten und die letzten Broadcasts aus. | | `analytics property create\|list`, `analytics geoip-update`, `workspace analytics-limit` | Websites für [Analytics](analytics.md), seine Standortdatenbank und das monatliche Event-Limit eines Workspaces ([Konfiguration](configuration.md#analytics)). | +| `workspace monitor-limit` | Monatliche Grenze eines Workspace für Request-Records des [API-Monitorings](analytics-api.md) ([Konfiguration](configuration.md#api-monitoring)). | | `person delete --email
--yes` | Löscht das Konto einer Person. Abgelehnt, solange sie einziger Inhaber eines Workspaces ist. | | `ingest-email --to
[--from
]` | Stellt eine Rohnachricht von stdin zu ([E-Mail](email.md#beliebiger-mta-yuva-ingest-email)). | | `vapid-keys` | Gibt ein neues Web-Push-Schlüsselpaar aus. Braucht keine Datenbank. | diff --git a/docs/headless.md b/docs/headless.md index e71879c1..e689f36b 100644 --- a/docs/headless.md +++ b/docs/headless.md @@ -109,7 +109,7 @@ A call without the needed scope answers `403 insufficient_scope`. | `workspace:manage` | Workspace settings and usage | | `migrations:manage` | [Moving a mailbox to Yuva](migrate.md): detect, create, connect, scan, start, finish, remove imported history. Never given to agents or to keys limited to inboxes. | | `analytics:read` | [Analytics](analytics.md#api): properties, numbers, realtime, health | -| `analytics:write` | Send [server-side events](analytics-server.md) (`POST /v1/analytics/properties/{propertyId}/events`). API keys only, never offered to OAuth clients. | +| `analytics:write` | Send [server-side events](analytics-server.md) (`POST /v1/analytics/properties/{propertyId}/events`) and [API request records](analytics-api.md) (`POST /v1/analytics/properties/{propertyId}/requests`). API keys only, never offered to OAuth clients. | | `analytics:manage` | Create, change and delete properties, rotate their keys, delete their data. Never given to agents or to keys limited to inboxes. | | `feedback:write` | `POST /v1/feedback` | diff --git a/docs/operations.md b/docs/operations.md index 8dd761ef..93c46023 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -83,6 +83,7 @@ Commands that print an id or a secret write only that to stdout. | `workspace broadcasts limit --workspace --daily ` | Sets the workspace's recipients per UTC day; `0` removes the limit. | | `workspace broadcasts show --workspace ` | Prints its status, limit, 30-day rates and latest broadcasts. | | `analytics property create\|list`, `analytics geoip-update`, `workspace analytics-limit` | Websites for [Analytics](analytics.md), its location database and a workspace's monthly event limit ([Configuration](configuration.md#analytics)). | +| `workspace monitor-limit` | A workspace's monthly limit of [API monitoring](analytics-api.md) request records ([Configuration](configuration.md#api-monitoring)). | | `person delete --email
--yes` | Deletes a person's account. Refused while they are a workspace's only owner. | | `ingest-email --to
[--from
]` | Delivers a raw message from stdin ([E-mail](email.md#any-mta-yuva-ingest-email)). | | `vapid-keys` | Prints a new Web Push key pair. Needs no database. | diff --git a/docs/tr/analytics-api.md b/docs/tr/analytics-api.md new file mode 100644 index 00000000..f95709a0 --- /dev/null +++ b/docs/tr/analytics-api.md @@ -0,0 +1,200 @@ +# API izleme + +Bir API özelliği, kendi API'nizin karşıladığı istekleri sayar: kaç tane olduğunu, kaçının başarısız olduğunu ve ne +kadar sürdüğünü, rota rota. Sunucunuz yanıtladığı her isteği bildirir; Yuva bunları dakika ve saat başına toplar, +istek başına satır, başlık, gövde ya da IP adresi saklamaz. Bu sayfa [Analitik](analytics.md) sayfasının +üzerine kuruludur. + +## API özelliği oluşturun + +API özelliği, `kind: api` olan bir analitik özelliğidir. Alan adı ve izleme betiği yoktur. Sahipler ve yöneticiler +onu `analytics:manage` kapsamlı bir anahtarla oluşturur: + +```sh +curl https://support.example.com/v1/analytics/properties \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "kind": "api", "name": "Shop API", "domains": [], "timezone": "Europe/Berlin" }' +``` + +Yanıt özelliğin `id` değerini taşır. Özelliğin saat dilimi, sayılarının hangi saat ve günlere göre sayılacağını +belirler; `time:minute` sorgusunun dakikaları da bu dilimdedir. + +Sonra kayıtları gönderecek anahtarı **Ayarlar → API anahtarları** altında `analytics:write` kapsamıyla oluşturun ve +sunucuda tutun. Bir üyenin oturumu reddedilir (`403 forbidden`); bir web sitesi özelliği `422 property_kind` +yanıtlar. `YUVA_MONITOR=off` ile başlatılan sunucuda API özelliği yoktur. + +## Go'dan istek gönderme + +```sh +go get github.com/productdevbook/yuva/sdk/go/analytics +``` + +`Middleware` bir `net/http` işleyicisini sarar ve yanıtladığı her isteği bildirir. Mux'ın eşleştirdiği kalıbın +bilinmesi için `http.ServeMux`'un kendisini sarın: `GET /users/{id}`, `/users/{id}` rotası olarak kaydedilir. + +```go +import "github.com/productdevbook/yuva/sdk/go/analytics" + +mux := http.NewServeMux() +mux.HandleFunc("GET /users/{id}", getUser) + +monitor, err := analytics.New(analytics.Config{ + BaseURL: "https://support.example.com", + APIKey: os.Getenv("YUVA_API_KEY"), + PropertyID: os.Getenv("YUVA_API_PROPERTY"), +}) +if err != nil { + log.Fatal(err) +} +defer monitor.Close(context.Background()) + +handler := monitor.Middleware(analytics.Monitor{ + Service: "shop-api", + Consumer: func(r *http.Request) string { return apiKeyName(r) }, +})(mux) +log.Fatal(http.ListenAndServe(":8080", handler)) +``` + +Kayıtlar, [sunucu tarafı olayların](analytics-server.md#go) yeniden denemeleriyle, 5 saniyede bir en çok 500'lük +gzip gruplar halinde gider. Süreç çıkmadan önce `Close` çağırın; yoksa bekleyen kayıtlar kaybolur. + +| `Monitor` | Varsayılan | | +|---|---|---| +| `Route` | mux'ın kalıbı | Yönlendiriciniz `ServeMux` değilse bir isteğin rotasını verir. | +| `Consumer` | yok | Kimin çağırdığı: bir API anahtarının adı ya da istemci kimliği. Hiçbir zaman bir kişinin adresi olmaz; e-posta biçimindeki değer `(scrubbed)` olarak saklanır. | +| `APIVersion` | yok | İsteğin kullandığı API sürümü. | +| `Environment` | `production` | `production`, `staging` gibi ortamları ayırır. | +| `Service` | yok | İsteği karşılayan servis. | +| `Release` | yok | Servisin sürümü. Kabul edilir, henüz gösterilmez. | +| `SampleRate` | her istek | `10`, başarılı isteklerin (durum 1-399) ondan birini gönderir; her biri on isteği temsil eder. Hatalar ve istemci hataları her zaman gönderilir. | +| `Country` | kapalı | Yuva'nın isteğin ülkesini sayabilmesi için arayanın adresini (`Config.ClientIP`) gönderir. Varışta kısaltılır ve hiç saklanmaz. | + +Hiçbir kalıbın eşleşmediği istek yoluyla gönderilir. Panik eden bir işleyici yazdığı durumla kaydedilir; hiçbir şey +yazmadıysa `0` ile. + +## curl ile istek gönderme + +Diğer tüm teknolojiler aynı kayıtları gönderir. `POST /v1/analytics/properties/{propertyId}/requests` 1 ile 1.000 +arası kayıt alır ve `202` yanıtlar. Gövde `Content-Encoding: gzip` ile sıkıştırılabilir. + +```sh +curl https://support.example.com/v1/analytics/properties/PROPERTY_ID/requests \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "records": [{ + "method": "GET", + "route": "/users/{id}", + "status": 200, + "duration_us": 18400, + "response_size": 612, + "consumer": "mobile-app", + "environment": "production" + }] + }' +``` + +```json +{ "accepted": 1, "errors": [] } +``` + +| Alan | | +|---|---| +| `method` | Zorunlu, 1 ile 20 karakter. RFC 9110 yöntemleri ve `PATCH` korunur; diğerleri `_OTHER` sayılır. | +| `route` | Yönlendiricinin eşleştirdiği kalıp; `/` ile başlar, en çok 300 karakter, yöntem ve alan adı olmadan. Bunu ya da `path` verin. | +| `path` | Yönlendiricinin kalıp bilmediği durumda isteğin yolu. Yalnızca ondan çıkarılan kalıp saklanır. | +| `status` | Zorunlu, 100 ile 599; yanıt gönderilmediyse `0`. | +| `duration_us` | Zorunlu, mikrosaniye cinsinden süre, en çok 24 saat. | +| `request_size`, `response_size` | Gövdelerin bayt sayısı. | +| `time` | İsteğin başladığı an, RFC 3339. Boşsa şimdi. | +| `consumer` | En çok 100 karakter; yukarıya bakın. | +| `api_version`, `environment`, `service`, `release` | Sırasıyla en çok 40, 40, 100 ve 80 karakter. Boş bırakılan `environment` `production` olur. | +| `ip` | Arayanın adresi, yalnızca ülkesini bulmak için. | +| `sample_rate` | 1 ile 10.000: bu kayıt o kadar isteği temsil eder. Yalnızca 1-399 durumu örneklenebilir; `sample_rate` 1'den büyük ve hata durumlu kayıt reddedilir. | + +## Rotalar + +Bir rota, API'nizin bir bölümünü adlandırır, hiçbir zaman tek bir kaydı; bu yüzden bir kimlik asla kendi başına rota +olmaz. + +- **Gönderdiğiniz rotalar temizlenir.** Bir `route` ya da `path` içinde kimliğe benzeyen her bölüm `{id}` olur: + sayılar, UUID'ler, tarihler, 16 ya da daha çok karakterlik onaltılık dizeler, en az iki rakam (ya da iki harf + büyüklüğü ve bir rakam) içeren 16 ya da daha çok harf, rakam, `-` ve `_` dizileri, e-posta biçimindeki değerler + ve 40 karakterden uzun her şey. Yönlendirici yer tutucusu gibi başlayan bölümler (`{id}`, `:id`, `*`, `[id]`, + ``) olduğu gibi kalır. Yani `/users/42` ve `/users/9f8e7d6c5b4a3210` ikisi de `/users/{id}` sayılır. +- **Kalıpsız 404 ya da 405 `(unmatched)` olur.** Yalnızca `path` veren ve `404` ya da `405` ile yanıtlanan kayıt + `(unmatched)` rotasında sayılır; böylece `/wp-login.php` yoklayan tarayıcılar rota üretmez. +- **Rota kuralları.** Özelliğin `path_rules` alanı, yukarıdaki temizlikten önce bir rotayı ya da yolu sizin + rotanıza eşler. Kural `{"pattern": "/legacy/**", "replacement": "/legacy"}` biçimindedir: `*` bir bölümle, `**` her + sayıda bölümle eşleşir, ilk eşleşen kural geçerlidir ve karşılık olduğu gibi kullanılır. + `PATCH /v1/analytics/properties/{propertyId}` ile ayarlanır. +- **Sınırlar.** Bir özellik en çok 1.000 rota, 10.000 tüketici, 50 API sürümü ve 20 ortam; bir çalışma alanı 100 + servis sayar. Sınırdan sonra yeni değerler `(other)` olarak sayılır ve sorgu yanıtı sınırı + `meta.caps_reached` içinde listeler. + +## Ne kaydedilir + +Dakika, saat ve gün başına kaydedilenler: istek sayısı, durumu 500-599 ya da `0` olanlar (hatalar) ve 400-499 +olanlar (istemci hataları), p50, p95 ve p99'u veren süre dağılımı; rota, yöntem, durum ve ortama göre. Servis, API +sürümü ve tüketici bunlarla birlikte saklanır. + +Hiçbir zaman kaydedilmeyen ve API'nin kabul etmediği şeyler: istek ya da yanıt gövdeleri, başlıklar, sorgu +dizeleri, parçalar (fragment) ve user agent'lar. Postgres'te istek başına satır yoktur. `ip` kısaltılır (IPv4 +`/24`, IPv6 `/48`), ülkesi bulunur ve aynı çağrıda atılır; hiçbir adres saklanmaz. Go ara katmanı adresi yalnızca +`Country: true` ile gönderir. + +## Sayıları okuma + +Panelde özelliğin **Genel bakış** sayfası zaman içinde istekleri, hata oranlarını ve gecikmeyi, **Rotalar** ise +rotaları trafikleri, hataları ve gecikmeleriyle gösterir. + +Aynı sayılar analitik sorgusundan gelir: `POST /v1/analytics/properties/{propertyId}/query`, `"dataset": "api"` +ile (kapsam `analytics:read`): + +```sh +curl https://support.example.com/v1/analytics/properties/PROPERTY_ID/query \ + -H "Authorization: Bearer $YUVA_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "dataset": "api", + "metrics": ["requests", "error_rate", "p95"], + "dimensions": ["route"], + "range": { "period": "24h" }, + "limit": 20 + }' +``` + +| | | +|---|---| +| Metrikler | `requests`, `requests_per_minute`, `errors`, `error_rate`, `client_errors`, `client_error_rate`, `p50`, `p95`, `p99`, `avg_duration`. Oranlar yüzde, süreler milisaniyedir. Yüzdelikler %1,1 doğrulukla verilir. | +| Boyutlar ve filtreler | `route`, `method`, `status`, `status_class` (`2xx`, `3xx`, `4xx`, `5xx`; `0` durumu için `none`), `environment`; en çok iki boyut, biri zaman aralığı olabilir. | +| Zaman aralıkları | `time:minute`, `time:hour`, `time:day`. Dakikalar ve saatler özelliğin yerel saatindedir. | +| Aralık | Geçerli dakikaya kadar `15m`, `1h`, `6h`, `24h`; ya da `today`, `7d`, `30d` ve diğer dönemler; ya da `from` ve `to`. | + +`time:minute`, [`YUVA_MONITOR_MINUTE_DAYS`](configuration.md#api-izleme) gün (varsayılan 8) geriye gider ve en çok +1.440 dakika döndürür; daha uzun ya da daha eski aralık `422 range_too_large` ile reddedilir. Saatlik sayılar +özelliğin `monitor_hour_days` değeri kadar (varsayılan 90, 7 ile 400 arası), günlük sayılar özelliğin +`retention_months` değeri kadar saklanır. `sample_rate` 1'den büyük gönderilen kayıtlara dayanan yanıtta +`meta.estimated: true` bulunur. Bir web sitesi özelliğinde `dataset: api` ve bir API özelliğinde bu alanı +vermeyen sorgu `422 property_kind` yanıtlar. API sorgusu `goal_id`, `prop`, `segment_id`, `compare` ya da +`insights` almaz. + +Bir kayıt sorguda yaklaşık 15 saniye içinde görünür. + +## Ne döner + +İstek geçerliyse, kayıtlara ne olduğundan bağımsız olarak yanıt `202`'dir. + +- **`errors`**, geçersiz diye dışarıda bırakılan kayıtları, her biri `records` içindeki `index`'i ve bir `detail` + ile listeler. Diğerleri alınır. +- **Alınır ama sayılmaz:** 72 saatten eski kayıtlar; özellik ya da anahtar için `YUVA_MONITOR_RECORDS_PER_MINUTE` + değerini aşan kayıtlar; çalışma alanının aylık sınırını aşan kayıtlar. +- **İsteğin tamamı reddedilir**: yukarıdaki biçimi bozan gövde (`400`), eksik ya da yanlış anahtar (`401`), + `analytics:write` kapsamı olmayan anahtar ya da üye oturumu (`403`), bilinmeyen özellik (`404`), web sitesi + özelliği (`422`) ve gzip dışındaki kodlama (`415`) için. İzin verdiği kadar açık satır tutan bir sunucu + `Retry-After` ile `503` yanıtlar: o süreden sonra grubu yeniden gönderin. + +Aylık sınır, kayıtların temsil ettiği istekleri değil, alınan kayıtları sayar: örneklenmiş bir kayıt `sample_rate` +değeri ne olursa olsun bir kayıttır. `GET /v1/analytics/properties`, ayın `requests` ve `requests_limit` +değerlerini `usage` altında gösterir. diff --git a/docs/tr/analytics.md b/docs/tr/analytics.md index be27342c..0278bcb3 100644 --- a/docs/tr/analytics.md +++ b/docs/tr/analytics.md @@ -271,7 +271,8 @@ Bir gelen kutusuna bağlı web sitesi ayrıca sayfalarının ne kadar destek do raporlarıyla gösterir: [Destek ve trafik](analytics-support.md). Aynı özellik iOS ve Android uygulamalarınızı ([Uygulamalarda analitik](analytics-apps.md)) ve arka ucunuzun -gönderdiği olayları ([Sunucu tarafı olaylar](analytics-server.md)) da sayar. +gönderdiği olayları ([Sunucu tarafı olaylar](analytics-server.md)) da sayar. `api` türündeki bir özellik ise kendi +API'nizin isteklerini sayar: [API izleme](analytics-api.md). **Kanallar** bir ziyaretin nereden geldiğini ayırır: Doğrudan, Organik arama, Ücretli arama, Sosyal, Ücretli sosyal, E-posta, Yönlendirme, **Yapay zekâ asistanları** (ChatGPT ve benzeri asistanlardan gelen diff --git a/docs/tr/configuration.md b/docs/tr/configuration.md index 847d15f0..98bc563a 100644 --- a/docs/tr/configuration.md +++ b/docs/tr/configuration.md @@ -168,6 +168,24 @@ Komutlar, [Operatör komutları](operations.md#operatör-komutları) bölümünd | `analytics geoip-update` | Bu ayın DB-IP Lite veritabanını, orada değilse `YUVA_GEOIP_DIR` içine indirir ve verilmesi gereken atfı yazdırır. `file` ile dosyanızı denetler. | | `workspace analytics-limit --workspace ` | Çalışma alanının aylık olay sayısını belirler; `0` sınırı kaldırır. | +## API izleme + +[API'nizi izleyin](analytics-api.md). Analitiğin parçasıdır: `YUVA_ANALYTICS=off` bunu da kapatır. + +| Değişken | Varsayılan | Anlamı | +|---|---|---| +| `YUVA_MONITOR` | `on` | `off`, API özelliklerini, istek kayıtlarını ve sorgularını kaldırır. | +| `YUVA_MONITOR_MONTHLY_REQUESTS` | `YUVA_SIGNUP=open` ise `10000000`, değilse `0` | Bir çalışma alanının UTC ayı başına gönderebileceği istek kaydı sayısı; fazlası ay bitene kadar sayılmaz. `0` sınır yok demektir. Çalışma alanı başına `yuva workspace monitor-limit` bunu geçersiz kılar. | +| `YUVA_MONITOR_RECORDS_PER_MINUTE` | `600000` | Bir API özelliğinin ve bir API anahtarının dakikada gönderebileceği istek kaydı sayısı; fazlası sayılmaz. En az 1. | +| `YUVA_MONITOR_BUFFER_KEYS` | `200000` | Bir işlemin yazmadan önce (10 saniyede bir) tuttuğu açık satır sayısı. O kadar satır açıkken uç nokta `Retry-After` ile `503` yanıtlar. En az 1000. | +| `YUVA_MONITOR_MINUTE_DAYS` | `8` | Dakika başına sayıların saklandığı gün sayısı, 2 ile 31 arası; `time:minute` sorguları o kadar geriye gider. | + +Komut, [Operatör komutları](operations.md#operatör-komutları) gibi çalıştırılır: + +| Komut | Ne yapar | +|---|---| +| `workspace monitor-limit --workspace ` | Çalışma alanının aylık istek kaydı sınırını ayarlar; `0` sınırı kaldırır. | + ## Sohbet | Değişken | Varsayılan | Anlamı | diff --git a/docs/tr/headless.md b/docs/tr/headless.md index 5bada5ec..48f5886c 100644 --- a/docs/tr/headless.md +++ b/docs/tr/headless.md @@ -112,7 +112,7 @@ Gereken yetki olmadan yapılan çağrı `403 insufficient_scope` döner. | `workspace:manage` | Çalışma alanı ayarları ve kullanım | | `migrations:manage` | [Bir posta kutusunu Yuva'ya taşıma](migrate.md): tespit, oluşturma, bağlama, tarama, başlatma, bitirme, aktarılan geçmişi kaldırma. Asistan hesaplarına ve gelen kutularıyla sınırlı anahtarlara asla verilmez. | | `analytics:read` | [Analitik](analytics.md#api): mülkler, rakamlar, canlı, durum | -| `analytics:write` | [Sunucu tarafı olayları](analytics-server.md) gönderme (`POST /v1/analytics/properties/{propertyId}/events`). Yalnızca API anahtarları; OAuth istemcilerine asla sunulmaz. | +| `analytics:write` | [Sunucu tarafı olayları](analytics-server.md) gönderme (`POST /v1/analytics/properties/{propertyId}/events`) ve [API istek kayıtları](analytics-api.md) (`POST /v1/analytics/properties/{propertyId}/requests`). Yalnızca API anahtarları; OAuth istemcilerine asla sunulmaz. | | `analytics:manage` | Mülk oluşturma, değiştirme ve silme, anahtarlarını yenileme, verilerini silme. Asistan hesaplarına ve gelen kutularıyla sınırlı anahtarlara asla verilmez. | | `feedback:write` | `POST /v1/feedback` | diff --git a/docs/tr/operations.md b/docs/tr/operations.md index 05dd0f82..c48e5786 100644 --- a/docs/tr/operations.md +++ b/docs/tr/operations.md @@ -84,6 +84,7 @@ Bir id ya da gizli anahtar yazdıran komutlar stdout'a yalnızca onu yazar. | `workspace broadcasts limit --workspace --daily ` | Çalışma alanının UTC günü başına alıcı sayısını belirler; `0` sınırı kaldırır. | | `workspace broadcasts show --workspace ` | Durumunu, sınırını, 30 günlük oranlarını ve son bültenlerini yazdırır. | | `analytics property create\|list`, `analytics geoip-update`, `workspace analytics-limit` | [Analitik](analytics.md) için web siteleri, konum veritabanı ve bir çalışma alanının aylık olay sınırı ([Yapılandırma](configuration.md#analitik)). | +| `workspace monitor-limit` | Bir çalışma alanının [API izleme](analytics-api.md) istek kaydı aylık sınırı ([Yapılandırma](configuration.md#api-izleme)). | | `person delete --email
--yes` | Bir kullanıcının hesabını siler. Bir çalışma alanının tek sahibiyse reddedilir. | | `ingest-email --to
[--from
]` | stdin'den gelen ham bir mesajı teslim eder ([E-posta](email.md#herhangi-bir-mta-yuva-ingest-email)). | | `vapid-keys` | Yeni bir Web Push anahtar çifti yazdırır. Veritabanı gerektirmez. | diff --git a/site/src/lib/docs.ts b/site/src/lib/docs.ts index 1b249257..0b0917fb 100644 --- a/site/src/lib/docs.ts +++ b/site/src/lib/docs.ts @@ -25,6 +25,7 @@ const groups = [ "analytics-events", "analytics-apps", "analytics-server", + "analytics-api", "analytics-sharing", "analytics-support", "analytics-consent", From 7c3d553dbb02b63e7e2cf79abbe72c1b3f5b623b Mon Sep 17 00:00:00 2001 From: productdevbook Date: Wed, 14 Oct 2026 13:53:11 +0200 Subject: [PATCH 08/69] feat(web): API properties with an overview, a routes table and a page per route An analytics property can now be added as an API. Its overview shows requests, requests per minute, error and client error rates, p95 and p99, the last 60 minutes by minute, requests, error rates and latency over time and the busiest routes; Routes lists every route and method sorted by traffic, errors or latency; a route's page has the same history and its status codes. Ranges include the last 15 minutes, hour, 6 hours and 24 hours. Reached caps and sampled (estimated) counts are shown, and so is the monthly request limit. Until the first request arrives, the property shows the Go middleware and a curl example for the records endpoint. en and tr. Refs #64 Co-Authored-By: Claude Opus 5.5 --- .../features/analytics/AnalyticsColumn.tsx | 35 +- .../features/analytics/AnalyticsRoutes.tsx | 114 ++- web/src/features/analytics/Controls.tsx | 18 +- web/src/features/analytics/Install.tsx | 102 +++ web/src/features/analytics/Monitor.tsx | 789 ++++++++++++++++++ web/src/features/analytics/NewProperty.tsx | 90 +- web/src/features/analytics/SharePage.tsx | 3 + web/src/features/analytics/parts.tsx | 9 +- web/src/features/analytics/queries.ts | 15 +- web/src/features/analytics/state.ts | 54 +- web/src/features/analytics/text.ts | 78 +- web/src/lib/schema.gen.ts | 285 ++++++- web/src/locales/en/analytics.po | 303 ++++++- web/src/locales/tr/analytics.po | 303 ++++++- 14 files changed, 2104 insertions(+), 94 deletions(-) create mode 100644 web/src/features/analytics/Monitor.tsx diff --git a/web/src/features/analytics/AnalyticsColumn.tsx b/web/src/features/analytics/AnalyticsColumn.tsx index 9817e23e..f7fc9f18 100644 --- a/web/src/features/analytics/AnalyticsColumn.tsx +++ b/web/src/features/analytics/AnalyticsColumn.tsx @@ -1,7 +1,7 @@ import "@/features/analytics/i18n" import { Plural, Trans, useLingui } from "@lingui/react/macro" -import { ChartLineIcon, GaugeIcon, PlusIcon } from "lucide-react" +import { ChartLineIcon, GaugeIcon, PlusIcon, ServerIcon } from "lucide-react" import { Link } from "react-router" import { Dot, ErrorLine, Notice } from "@/components/common" @@ -47,13 +47,15 @@ export function UsageBanner({ usage, className }: { usage: AnalyticsUsage; class function Row({ p, now, inboxColor }: { p: AnalyticsProperty; now: number; inboxColor?: string }) { const { t, i18n } = useLingui() const search = useViewSearch() - const quiet = !p.last_event_at || now - Date.parse(p.last_event_at) > DAY + const api = p.kind === "api" + const quiet = !api && (!p.last_event_at || now - Date.parse(p.last_event_at) > DAY) const visitors = formatCompact(p.visitors_today, i18n.locale) + const Icon = api ? ServerIcon : ChartLineIcon return (
  • - + @@ -67,16 +69,22 @@ function Row({ p, now, inboxColor }: { p: AnalyticsProperty; now: number; inboxC /> )} - - {p.domains.join(", ")} - + {api && p.domains.length === 0 ? ( + {t`API`} + ) : ( + + {p.domains.join(", ")} + + )} - - {visitors} - - + {!api && ( + + {visitors} + + + - + )}
    • ) @@ -119,7 +127,10 @@ export default function AnalyticsColumn() {

    {canManage ? (

    - Add a website to count its visitors, pages and sources, without cookies. + + Add a website to count its visitors, pages and sources without cookies, or an API to count its requests + per route. +

    ) : (

    diff --git a/web/src/features/analytics/AnalyticsRoutes.tsx b/web/src/features/analytics/AnalyticsRoutes.tsx index b21ea318..a7ff8b75 100644 --- a/web/src/features/analytics/AnalyticsRoutes.tsx +++ b/web/src/features/analytics/AnalyticsRoutes.tsx @@ -1,7 +1,7 @@ import "@/features/analytics/i18n" import { Trans, useLingui } from "@lingui/react/macro" -import { ChartLineIcon, PlusIcon } from "lucide-react" +import { ChartLineIcon, PlusIcon, ServerIcon } from "lucide-react" import { Link, Navigate, Outlet, Route, Routes, useLocation, useNavigate, useParams } from "react-router" import { Dot, ErrorLine } from "@/components/common" @@ -14,8 +14,10 @@ import { Controls } from "@/features/analytics/Controls" import { EventsView } from "@/features/analytics/Events" import { FunnelsView } from "@/features/analytics/Funnels" import { GoalsView } from "@/features/analytics/Goals" +import { ApiOverview, ApiRoutePage, ApiRoutes, RequestsBanner } from "@/features/analytics/Monitor" import { InstallPropertyPage, NewPropertyPage, WaitForFirstEvent } from "@/features/analytics/NewProperty" import { Overview } from "@/features/analytics/Overview" +import { useCurrentProperty } from "@/features/analytics/parts" import { useAnalyticsProperties } from "@/features/analytics/properties" import { useProperty } from "@/features/analytics/queries" import { Realtime } from "@/features/analytics/Realtime" @@ -26,6 +28,7 @@ import { useViewSearch } from "@/features/analytics/state" import { useAnalyticsText } from "@/features/analytics/text" import { Segmented } from "@/components/common" import { useIsPhone } from "@/hooks/use-media-query" +import type { AnalyticsProperty } from "@/lib/api" import { useSession } from "@/lib/session" import { useInboxes } from "@/lib/workspace" @@ -45,6 +48,9 @@ const VIEWS = [ ] as const type ViewKey = (typeof VIEWS)[number] +const API_VIEWS = ["overview", "routes"] as const +type ApiViewKey = (typeof API_VIEWS)[number] + function AnalyticsHome() { const { canManage } = useSession() const phone = useIsPhone() @@ -60,10 +66,16 @@ function AnalyticsHome() {

    {canManage && ( - +
    + + +
    )} ) @@ -78,7 +90,6 @@ function PropertyLayout() { const search = useViewSearch(true) const property = useProperty(propertyId) const usage = useAnalyticsProperties().data?.usage - const inbox = (useInboxes().data ?? []).find((i) => i.id === property.data?.inbox_id) const last = pathname.split("/")[3] as ViewKey | undefined const current: ViewKey = last && VIEWS.includes(last) ? last : "overview" const names: Record = { @@ -104,28 +115,12 @@ function PropertyLayout() { ) } + if (p.kind === "api") return + if ((last as string | undefined) === "routes") return const controls = current !== "realtime" && current !== "settings" && current !== "retention" return ( -
    - -

    - {p.name} -

    -

    - {inbox && ( - - - {inbox.name} - - )} - - {p.domains.join(", ")} - - · - {p.timezone} -

    -
    + {usage && }
    : +} + +function PropertyHeader({ property: p }: { property: AnalyticsProperty }) { + const { t } = useLingui() + const inbox = (useInboxes().data ?? []).find((i) => i.id === p.inbox_id) + return ( +
    + +

    + {p.name} +

    +

    + {inbox && ( + + + {inbox.name} + + )} + {p.kind === "api" && ( + + + API + + )} + {p.domains.length > 0 && ( + + {p.domains.join(", ")} + + )} + · + {p.timezone} +

    +
    + ) +} + +function ApiLayout({ property: p }: { property: AnalyticsProperty }) { + const { t } = useLingui() + const { pathname } = useLocation() + const navigate = useNavigate() + const search = useViewSearch() + const usage = useAnalyticsProperties().data?.usage + const last = pathname.split("/")[3] as ApiViewKey | undefined + if (last && !API_VIEWS.includes(last)) return + const current: ApiViewKey = last ?? "overview" + const names: Record = { overview: t`Overview`, routes: t`Routes` } + return ( + + + {usage && } +
    + navigate(`/analytics/${p.id}${v === "overview" ? "" : `/${v}`}${search}`)} + label={t`Analytics views`} + items={API_VIEWS.map((v) => ({ value: v, label: names[v], testId: `analytics-tab-${v}` }))} + /> +
    + +     + ) +} + export default function AnalyticsRoutes() { return ( @@ -155,7 +215,7 @@ export default function AnalyticsRoutes() { } /> } /> }> - } /> + } /> } /> } /> } /> @@ -167,6 +227,8 @@ export default function AnalyticsRoutes() { } /> } /> } /> + } /> + } /> } /> diff --git a/web/src/features/analytics/Controls.tsx b/web/src/features/analytics/Controls.tsx index 2db86fcd..c5c05071 100644 --- a/web/src/features/analytics/Controls.tsx +++ b/web/src/features/analytics/Controls.tsx @@ -31,7 +31,17 @@ import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" import { useDeleteSegment, useSaveSegment, useSegments } from "@/features/analytics/queries" -import { OPS, PERIODS, sameFilters, todayIso, useView, type FilterOp, type Period } from "@/features/analytics/state" +import { + API_PERIODS, + OPS, + PERIODS, + sameFilters, + todayIso, + useView, + type FilterOp, + type Kind, + type Period, +} from "@/features/analytics/state" import { flag, useAnalyticsText, valueText, type AnalyticsText } from "@/features/analytics/text" import { Field } from "@/features/settings/ui" import { ApiError, type AnalyticsFilter, type AnalyticsSegment } from "@/lib/api" @@ -103,10 +113,10 @@ function daysAgo(n: number) { return d.toISOString().slice(0, 10) } -export function RangePicker() { +export function RangePicker({ kind = "site" }: { kind?: Kind }) { const { t, i18n } = useLingui() const text = useAnalyticsText() - const { view, setRange } = useView() + const { view, setRange } = useView(kind) const [custom, setCustom] = useState(false) const r = view.range const customLabel = r.period @@ -128,7 +138,7 @@ export function RangePicker() { - {PERIODS.map((p) => ( + {(kind === "api" ? API_PERIODS : PERIODS).map((p) => ( {text.period[p]} diff --git a/web/src/features/analytics/Install.tsx b/web/src/features/analytics/Install.tsx index eafa873e..b4043dc6 100644 --- a/web/src/features/analytics/Install.tsx +++ b/web/src/features/analytics/Install.tsx @@ -424,3 +424,105 @@ export function InstallSnippets({ property }: { property: AnalyticsProperty }) {
    ) } + +type ApiTab = "go" | "curl" + +export function apiSnippet(tab: ApiTab, v: { server: string; id: string }) { + if (tab === "go") { + return [ + `// go get github.com/productdevbook/yuva/sdk/go/analytics`, + `import "github.com/productdevbook/yuva/sdk/go/analytics"`, + ``, + `yuva, err := analytics.New(analytics.Config{`, + `\tBaseURL: "${v.server}",`, + `\tAPIKey: os.Getenv("YUVA_API_KEY"), // a key with analytics:write`, + `\tPropertyID: "${v.id}",`, + `})`, + `if err != nil {`, + `\tlog.Fatal(err)`, + `}`, + `defer yuva.Close(context.Background())`, + ``, + `mux := http.NewServeMux()`, + `mux.HandleFunc("GET /users/{id}", getUser)`, + ``, + `// wrap the mux itself, so the route pattern it matched is recorded`, + `handler := yuva.Middleware(analytics.Monitor{`, + `\tService: "shop",`, + `\tEnvironment: "production",`, + `})(mux)`, + `log.Fatal(http.ListenAndServe(":8080", handler))`, + ].join("\n") + } + return [ + `curl ${v.server}/v1/analytics/properties/${v.id}/requests \\`, + ` -H "Authorization: Bearer $YUVA_API_KEY" \\`, + ` -H "Content-Type: application/json" \\`, + ` -d '{`, + ` "records": [{`, + ` "method": "GET",`, + ` "route": "/users/{id}",`, + ` "status": 200,`, + ` "duration_us": 12500`, + ` }]`, + ` }'`, + ``, + `# {"accepted": 1, "errors": []}`, + ].join("\n") +} + +function ApiHint({ tab }: { tab: ApiTab }) { + const { canManage } = useSession() + if (tab === "curl") { + return ( + + Up to 1,000 records per request, gzip allowed. Send route when your router knows the pattern, else path, and ids + are taken out of it. Records left out as invalid come back in errors with their position. + + ) + } + return canManage ? ( + + Middleware records every request the mux answers, with its route pattern, status and duration, and sends them in + batches. It needs an{" "} + + API key + {" "} + with the analytics:write scope. + + ) : ( + + Middleware records every request the mux answers, with its route pattern, status and duration, and sends them in + batches. It needs an API key with the analytics:write scope from an owner or admin. + + ) +} + +export function ApiInstallSnippets({ property }: { property: AnalyticsProperty }) { + const { t } = useLingui() + const [tab, setTab] = useState("go") + return ( +
    +
    + +
    +

    + +

    +
    + +
    +
    + ) +} diff --git a/web/src/features/analytics/Monitor.tsx b/web/src/features/analytics/Monitor.tsx new file mode 100644 index 00000000..e8c30e3a --- /dev/null +++ b/web/src/features/analytics/Monitor.tsx @@ -0,0 +1,789 @@ +import { Plural, Trans, useLingui } from "@lingui/react/macro" +import { ArrowDownIcon, ArrowLeftIcon, GaugeIcon, LayersIcon, SigmaIcon } from "lucide-react" +import { Link, useParams, useSearchParams } from "react-router" +import { Area, AreaChart, Bar, BarChart, CartesianGrid, XAxis, YAxis } from "recharts" + +import { Notice, Segmented } from "@/components/common" +import { ChartContainer, ChartTooltip, ChartTooltipContent, type ChartConfig } from "@/components/ui/chart" +import { Button } from "@/components/ui/button" +import { Skeleton } from "@/components/ui/skeleton" +import { RangePicker } from "@/features/analytics/Controls" +import { ApiInstallSnippets } from "@/features/analytics/Install" +import { BarList, CsvButton, Panel, PanelProblem, QueryProblem, useCurrentProperty } from "@/features/analytics/parts" +import { downloadCsv, useAnalyticsTable, useApiQuery } from "@/features/analytics/queries" +import { apiGrainFor, useView, useViewSearch } from "@/features/analytics/state" +import { + formatCompact, + formatMetric, + num, + OTHER, + useAnalyticsText, + valueText, + type NumMetric, +} from "@/features/analytics/text" +import type { AnalyticsDimension, AnalyticsFilter, AnalyticsQuery, AnalyticsResult, AnalyticsUsage } from "@/lib/api" +import { cn } from "@/lib/utils" + +type Cap = NonNullable[number] + +const KPIS: NumMetric[] = ["requests", "requests_per_minute", "error_rate", "client_error_rate", "p95", "p99"] + +function useApiView() { + return useView("api") +} + +function routeFilters(route?: string, method?: string): AnalyticsFilter[] { + const out: AnalyticsFilter[] = [] + if (route !== undefined) out.push({ dimension: "route", op: "is", values: [route] }) + if (method) out.push({ dimension: "method", op: "is", values: [method] }) + return out +} + +export function routePath(propertyId: string, method: string, route: string) { + const tail = route.startsWith("/") ? route.slice(1) : route + return `/analytics/${propertyId}/routes/${encodeURIComponent(method)}/${tail.split("/").map(encodeURIComponent).join("/")}` +} + +function bucketDate(v: string) { + return new Date(v.length > 10 ? `${v}:00Z` : `${v}T00:00:00Z`) +} + +export function RequestsBanner({ usage }: { usage: AnalyticsUsage }) { + const { i18n } = useLingui() + const used = usage.requests ?? 0 + const cap = usage.requests_limit + if (!cap) return null + const fmt = new Intl.NumberFormat(i18n.locale) + const limit = fmt.format(cap) + if (used >= cap) { + return ( + + + This workspace reached its {limit} API requests for this month. Further requests are not counted until the + month ends. + + + ) + } + if (used < cap * 0.8) return null + const count = fmt.format(used) + return ( + + + {count} of {limit} API requests counted this month. Requests over the limit are not counted until the month + ends. + + + ) +} + +function CapLine({ cap }: { cap: Cap }) { + const { i18n } = useLingui() + const n = (v: number) => new Intl.NumberFormat(i18n.locale).format(v) + switch (cap) { + case "routes": { + const max = n(1000) + return This API reached {max} routes. New routes are counted under Other. + } + case "consumers": { + const max = n(10000) + return This API reached {max} consumers. New consumers are counted under Other. + } + case "api_versions": { + const max = n(50) + return This API reached {max} API versions. New versions are counted under Other. + } + case "app_versions": { + const max = n(200) + return This API reached {max} app versions. New versions are counted under Other. + } + case "environments": { + const max = n(20) + return This API reached {max} environments. New environments are counted under Other. + } + case "services": { + const max = n(100) + return The workspace reached {max} services. New services are counted under Other. + } + case "dimension_rows": { + const max = n(50000) + return ( + + An hour of this range reached {max} rows of consumers, countries or app versions. New ones in that hour are + counted under Other. + + ) + } + } +} + +function CapsNotice({ caps }: { caps?: Cap[] }) { + if (!caps?.length) return null + return ( + +
      + {caps.map((c) => ( +
    • + +
      • + ))} +
    +     + ) +} + +function Estimated({ show }: { show?: boolean }) { + const { t } = useLingui() + if (!show) return null + return ( + + + Estimated + + ) +} + +function ApiControls({ meta }: { meta?: AnalyticsResult["meta"] }) { + return ( +
    +
    + + +
    + +
    + ) +} + +function Kpis({ filters, testId }: { filters: AnalyticsFilter[]; testId: string }) { + const { i18n } = useLingui() + const text = useAnalyticsText() + const property = useCurrentProperty() + const { view } = useApiView() + const q = useApiQuery(property.id, { metrics: KPIS, range: view.range, filters }) + return ( + <> + + +
    + {KPIS.map((m) => ( +
    + {text.metric[m]} + {q.isPending ? ( + + ) : ( + {formatMetric(m, num(q.data?.totals, m), i18n.locale)} + )} +
    + ))} +
    + + ) +} + +const SERIES_COLOR: Partial> = { + requests: "var(--brand)", + errors: "var(--destructive)", + error_rate: "var(--destructive)", + client_error_rate: "var(--warning)", + p50: "var(--muted-foreground)", + p95: "var(--brand)", + p99: "var(--warning)", +} + +function grainText(grain: AnalyticsDimension) { + if (grain === "time:minute") return per minute + if (grain === "time:hour") return per hour + return per day +} + +function SeriesChart({ + title, + metrics, + filters, + testId, +}: { + title: React.ReactNode + metrics: NumMetric[] + filters: AnalyticsFilter[] + testId: string +}) { + const { i18n } = useLingui() + const text = useAnalyticsText() + const property = useCurrentProperty() + const { view } = useApiView() + const grain = apiGrainFor(view.range) + const q = useApiQuery(property.id, { metrics, dimensions: [grain], range: view.range, filters }) + const short = grain !== "time:day" + const tick = new Intl.DateTimeFormat( + i18n.locale, + short ? { timeStyle: "short", timeZone: "UTC" } : { day: "numeric", month: "short", timeZone: "UTC" } + ) + const long = new Intl.DateTimeFormat(i18n.locale, { + dateStyle: "medium", + ...(short ? { timeStyle: "short" } : {}), + timeZone: "UTC", + }) + const data = (q.data?.rows ?? []).map((r) => { + const row: Record = { at: r.dimensions[0] ?? "" } + for (const m of metrics) row[m] = num(r.metrics, m) ?? 0 + return row + }) + const config: ChartConfig = Object.fromEntries( + metrics.map((m) => [m, { label: text.metric[m], color: SERIES_COLOR[m] ?? "var(--brand)" }]) + ) + return ( + + {title} + {grainText(grain)} + + } + action={ + metrics.length > 1 && ( + + {metrics.map((m) => ( + + + {text.metric[m]} + + ))} + + ) + } + testId={testId} + > + {q.isPending ? ( + + ) : q.error ? ( + + ) : ( + + + + tick.format(bucketDate(String(v)))} + /> + formatMetric(metrics[0]!, Number(v), i18n.locale)} + /> + (p?.[0] ? long.format(bucketDate(String(p[0].payload.at))) : "")} + formatter={(value, name) => ( +
    + + {config[String(name)]?.label} + + {formatMetric(name as NumMetric, Number(value), i18n.locale)} + +
    + )} + /> + } + /> + {metrics.map((m, i) => ( + + ))} +     +     + )} +     + ) +} + +function LastHour({ filters }: { filters: AnalyticsFilter[] }) { + const { i18n } = useLingui() + const text = useAnalyticsText() + const property = useCurrentProperty() + const q = useApiQuery(property.id, { + metrics: ["requests", "errors"], + dimensions: ["time:minute"], + range: { period: "1h" }, + filters, + }) + const time = new Intl.DateTimeFormat(i18n.locale, { timeStyle: "short", timeZone: "UTC" }) + const data = (q.data?.rows ?? []).map((r) => { + const requests = num(r.metrics, "requests") ?? 0 + const errors = num(r.metrics, "errors") ?? 0 + return { at: r.dimensions[0] ?? "", ok: Math.max(0, requests - errors), errors, requests } + }) + const config: ChartConfig = { + ok: { label: text.metric.requests, color: "var(--brand)" }, + errors: { label: text.metric.errors, color: "var(--destructive)" }, + } + const fmt = (v: number) => formatMetric("requests", v, i18n.locale) + const requests = num(q.data?.totals, "requests") ?? 0 + const errors = num(q.data?.totals, "errors") ?? 0 + return ( + The last 60 minutes} + action={ + q.data && ( + + + {" · "} + + + ) + } + testId="api-last-hour" + > + {q.isPending ? ( + + ) : q.error ? ( + + ) : ( + + + + time.format(bucketDate(String(v)))} + /> + fmt(v)} /> + (p?.[0] ? time.format(bucketDate(String(p[0].payload.at))) : "")} + /> + } + /> + + + + + )} + + ) +} + +function MethodTag({ method }: { method: string }) { + const text = useAnalyticsText() + const { i18n } = useLingui() + return ( + + {method === "_OTHER" ? valueText(text, "method", method, i18n.locale) : method} + + ) +} + +function RouteName({ method, route }: { method?: string; route: string }) { + const { i18n } = useLingui() + const text = useAnalyticsText() + const plain = route.startsWith("/") + return ( + + {method && } + + {valueText(text, "route", route, i18n.locale)} + + + ) +} + +function TopRoutes() { + const { i18n } = useLingui() + const text = useAnalyticsText() + const property = useCurrentProperty() + const search = useViewSearch() + const { view } = useApiView() + const q = useApiQuery(property.id, { + metrics: ["requests"], + dimensions: ["route", "method"], + range: view.range, + limit: 8, + }) + const rows = q.data?.rows ?? [] + return ( + Busiest routes} + testId="api-top-routes" + footer={ + + All routes + + } + > + {q.isPending ? ( + + ) : q.error ? ( + + ) : rows.length === 0 ? ( +

    + No requests in this range. +

    + ) : ( + { + const [route = "", method = ""] = r.dimensions + const v = num(r.metrics, "requests") ?? 0 + return { + key: `${method} ${route} ${i}`, + label: , + value: v, + display: formatCompact(v, i18n.locale), + to: route === OTHER ? undefined : `${routePath(property.id, method, route)}${search}`, + } + })} + /> + )} +     + ) +} + +function useHasRequests(propertyId: string) { + const q = useApiQuery(propertyId, { metrics: ["requests"], range: { period: "12mo" } }) + return { pending: q.isPending, error: q.error, any: (num(q.data?.totals, "requests") ?? 0) > 0 } +} + +function ApiEmpty() { + const property = useCurrentProperty() + return ( +
    + +

    + No requests yet +

    +

    + + Add the middleware to your server or send records with any HTTP client. Requests show up here about 15 + seconds after your server answered them. + +

    +     +
    + +
    +
    + ) +} + +function WhenRequests({ children }: { children: React.ReactNode }) { + const property = useCurrentProperty() + const has = useHasRequests(property.id) + if (has.pending) return + if (has.error) return + if (!has.any) return + return <>{children} +} + +function Charts({ filters }: { filters: AnalyticsFilter[] }) { + const { t } = useLingui() + return ( + <> + +
    + + +
    + + ) +} + +export function ApiOverview() { + return ( + +
    + + + + +
    +     + ) +} + +type Sort = "traffic" | "errors" | "latency" + +const SORTS: Record = { + traffic: { metric: "requests", order: [{ metric: "requests", dir: "desc" }] }, + errors: { + metric: "error_rate", + order: [ + { metric: "error_rate", dir: "desc" }, + { metric: "errors", dir: "desc" }, + ], + }, + latency: { metric: "p95", order: [{ metric: "p95", dir: "desc" }] }, +} + +const TABLE: NumMetric[] = ["requests", "error_rate", "client_error_rate", "p50", "p95", "p99"] + +export function ApiRoutes() { + const { t, i18n } = useLingui() + const text = useAnalyticsText() + const property = useCurrentProperty() + const search = useViewSearch() + const { view } = useApiView() + const [params, setParams] = useSearchParams() + const raw = params.get("sort") as Sort | null + const sort: Sort = raw && raw in SORTS ? raw : "traffic" + const body: AnalyticsQuery = { + dataset: "api", + metrics: TABLE, + dimensions: ["route", "method"], + range: view.range, + order_by: SORTS[sort].order, + limit: 100, + } + const table = useAnalyticsTable(property.id, body) + const rows = table.data?.pages.flatMap((p) => p.rows) ?? [] + const meta = table.data?.pages[0]?.meta + const phoneMetric = SORTS[sort].metric + const names: Record = { traffic: t`Traffic`, errors: t`Errors`, latency: t`Latency` } + return ( + +
    + +
    +
    + + setParams( + (prev) => { + const next = new URLSearchParams(prev) + if (v === "traffic") next.delete("sort") + else next.set("sort", v) + return next + }, + { replace: true } + ) + } + label={t`Sort routes by`} + items={(Object.keys(SORTS) as Sort[]).map((s) => ({ + value: s, + label: names[s], + testId: `api-sort-${s}`, + }))} + /> +
    + downloadCsv(property.id, body, `${property.name}-routes`)} /> +
    + +
    + + + + + {TABLE.map((m) => ( + + ))} + + + + {table.isPending ? ( + + + + ) : rows.length === 0 ? ( + + + + ) : ( + rows.map((r, i) => { + const [route = "", method = ""] = r.dimensions + return ( + + + {TABLE.map((m) => ( + + ))} + + ) + }) + )} + +
    {text.dimension.route} + + {text.metric[m]} + {m === SORTS[sort].metric && } + +
    + +
    + No requests in this range. +
    + {route === OTHER ? ( + + ) : ( + + + + )} + + {formatMetric(m, num(r.metrics, m), i18n.locale)} +
    + {table.hasNextPage && ( +
    + +
    + )} +
    +
    +     + ) +} + +function statusTone(code: string) { + if (code === "0" || code.startsWith("5")) return "bg-destructive" + if (code.startsWith("4")) return "bg-warning" + if (code.startsWith("2")) return "bg-success" + return "bg-muted-foreground" +} + +function StatusCodes({ filters }: { filters: AnalyticsFilter[] }) { + const { i18n } = useLingui() + const text = useAnalyticsText() + const property = useCurrentProperty() + const { view } = useApiView() + const q = useApiQuery(property.id, { + metrics: ["requests"], + dimensions: ["status"], + range: view.range, + filters, + limit: 20, + }) + const rows = q.data?.rows ?? [] + const total = num(q.data?.totals, "requests") ?? 0 + const pct = new Intl.NumberFormat(i18n.locale, { style: "percent", maximumFractionDigits: 1 }) + return ( + Status codes} testId="api-status-codes"> + {q.isPending ? ( + + ) : q.error ? ( + + ) : rows.length === 0 ? ( +

    + No requests in this range. +

    + ) : ( + { + const code = r.dimensions[0] ?? "" + const v = num(r.metrics, "requests") ?? 0 + return { + key: code, + label: ( + + + {valueText(text, "status", code, i18n.locale)} + + ), + value: v, + display: `${formatCompact(v, i18n.locale)} · ${pct.format(total ? v / total : 0)}`, + } + })} + /> + )} +     + ) +} + +export function ApiRoutePage() { + const property = useCurrentProperty() + const search = useViewSearch() + const { method = "", "*": tail = "" } = useParams() + const route = tail.startsWith("(") ? tail : `/${tail}` + const filters = routeFilters(route, method) + return ( +
    +
    + + + All routes + +

    + +

    +
    + + +
    + + +
    +
    + ) +} diff --git a/web/src/features/analytics/NewProperty.tsx b/web/src/features/analytics/NewProperty.tsx index 4939064e..a6381d74 100644 --- a/web/src/features/analytics/NewProperty.tsx +++ b/web/src/features/analytics/NewProperty.tsx @@ -1,9 +1,9 @@ import { Trans, useLingui } from "@lingui/react/macro" import { CheckCircle2Icon } from "lucide-react" import { useMemo, useState } from "react" -import { Link, Navigate, useNavigate, useParams } from "react-router" +import { Link, Navigate, useNavigate, useParams, useSearchParams } from "react-router" -import { ErrorLine, Notice } from "@/components/common" +import { ErrorLine, Notice, Segmented } from "@/components/common" import { Pane, PaneBack } from "@/components/common/Column" import { formatDateTime } from "@/components/common/text" import { Button } from "@/components/ui/button" @@ -11,8 +11,9 @@ import { Input } from "@/components/ui/input" import { Skeleton } from "@/components/ui/skeleton" import { Spinner } from "@/components/ui/spinner" import { Textarea } from "@/components/ui/textarea" -import { InstallSnippets } from "@/features/analytics/Install" +import { ApiInstallSnippets, InstallSnippets } from "@/features/analytics/Install" import { useCreateProperty, useProperty, useWaitForFirstEvent } from "@/features/analytics/queries" +import type { Kind } from "@/features/analytics/state" import { ChoiceSelect, Field, FormCard, PageHeader, Section } from "@/features/settings/ui" import { useSession } from "@/lib/session" import { useInboxes } from "@/lib/workspace" @@ -42,8 +43,10 @@ export function NewPropertyPage() { const { t } = useLingui() const { canManage } = useSession() const navigate = useNavigate() + const [params] = useSearchParams() const inboxes = useInboxes().data ?? [] const zones = useZones() + const [kind, setKind] = useState(params.get("kind") === "api" ? "api" : "site") const [name, setName] = useState("") const [domains, setDomains] = useState("") const [inbox, setInbox] = useState(NO_INBOX) @@ -51,21 +54,41 @@ export function NewPropertyPage() { const create = useCreateProperty() if (!canManage) return const hosts = splitLines(domains).map(hostOnly).filter(Boolean) + const api = kind === "api" return ( Add a website} + title={api ? Add an API : Add a website} description={ - - Count visitors, pages, sources, places and devices without cookies and without storing IP addresses. - + api ? ( + + Count your API’s requests, errors and response times per route. Your server sends a short record per + request; no body, header, query string or IP address is kept. + + ) : ( + + Count visitors, pages, sources, places and devices without cookies and without storing IP addresses. + + ) } /> +
    + +
    create.mutate( { + kind, name: name.trim(), domains: hosts, inbox_id: inbox === NO_INBOX ? undefined : inbox, @@ -80,7 +103,7 @@ export function NewPropertyPage() {