From 9003ae05bf6c58b548b874b330f9062a83ec0684 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Sat, 24 Oct 2026 13:57:09 -0400 Subject: [PATCH 01/19] feat: add captureAi on the capture v1 AI endpoint `captureAi` takes the same arguments as `capture`, goes through the same enrichment, precedence, personless default and `beforeSend`, and queues the event on its own disk queue, executor and retry identity for `POST /i/v1/ai/events`. It returns the event's uuid once it's handed to the queue, or null when the SDK is disabled or opted out or `beforeSend` dropped it. A name not starting with `$ai_` logs one debug line and is sent unchanged; `capture` never reroutes `$ai_*` events. The AI request client uses `captureAiTimeout` for its call, read and write timeouts and honors `captureAiCompression` through its own gzip interceptor instance. `PostHogQueue` becomes byte-aware through `EndpointSpec`: `maxBatchBytes` bounds a request window (check before append, an oversize record goes alone) and flushes as soon as that much is queued; `maxQueueBytes` is a FIFO disk cap on top of `maxQueueSize`, enforced on add and on load; `maxRecordBytes` drops a record over the limit when it's queued, logging only its name and size. The analytics spec gets a 5 MiB request cap; the AI spec gets 5 MiB requests, 50 MiB on disk and `captureAiMaxEventBytes` (at most 8 MiB) plus a 64 KiB envelope per event. New `PostHogConfig` knobs: `captureAiCompression`, `captureAiMaxQueueSize`, `captureAiTimeout` and `captureAiMaxEventBytes`; non-positive values use the defaults. `PostHogAndroid` stores the AI queue under `posthog-disk-ai-queue`. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DRbQPidntd4hVufNpMT19L --- .../com/posthog/android/PostHogAndroid.kt | 1 + posthog/api/posthog.api | 22 +++ posthog/src/main/java/com/posthog/PostHog.kt | 147 +++++++++++++++--- .../main/java/com/posthog/PostHogConfig.kt | 55 +++++++ .../main/java/com/posthog/PostHogInterface.kt | 46 ++++++ .../java/com/posthog/internal/EndpointSpec.kt | 78 ++++++++++ .../internal/GzipRequestInterceptor.kt | 115 +++++++------- .../java/com/posthog/internal/PostHogApi.kt | 41 ++++- .../java/com/posthog/internal/PostHogQueue.kt | 104 +++++++++++-- .../java/com/posthog/PostHogFake.kt | 16 ++ 10 files changed, 536 insertions(+), 89 deletions(-) diff --git a/posthog-android/src/main/java/com/posthog/android/PostHogAndroid.kt b/posthog-android/src/main/java/com/posthog/android/PostHogAndroid.kt index c6e845b6e..d4b1bf6b4 100644 --- a/posthog-android/src/main/java/com/posthog/android/PostHogAndroid.kt +++ b/posthog-android/src/main/java/com/posthog/android/PostHogAndroid.kt @@ -195,6 +195,7 @@ public class PostHogAndroid private constructor() { config.storagePrefix = config.storagePrefix ?: File(cacheDir, "posthog-disk-queue").absolutePath config.replayStoragePrefix = config.replayStoragePrefix ?: File(cacheDir, "posthog-disk-replay-queue").absolutePath config.logsStoragePrefix = config.logsStoragePrefix ?: File(cacheDir, "posthog-disk-logs-queue").absolutePath + config.aiStoragePrefix = config.aiStoragePrefix ?: File(cacheDir, "posthog-disk-ai-queue").absolutePath val preferences = config.cachePreferences ?: PostHogSharedPreferences(context, config) config.cachePreferences = preferences // Defaults to PostHogDeviceDateProvider when api < 33 diff --git a/posthog/api/posthog.api b/posthog/api/posthog.api index f99193f55..bdad93b91 100644 --- a/posthog/api/posthog.api +++ b/posthog/api/posthog.api @@ -25,6 +25,7 @@ public final class com/posthog/PostHog : com/posthog/PostHogStateless, com/posth public fun addExceptionStep (Ljava/lang/String;Ljava/util/Map;)V public fun alias (Ljava/lang/String;Ljava/util/Map;)V public fun capture (Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;)V + public fun captureAi (Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;)Ljava/util/UUID; public fun captureException (Ljava/lang/Throwable;Ljava/util/Map;Ljava/util/Map;)V public fun captureFeatureInteraction (Ljava/lang/String;Ljava/lang/String;)V public fun captureFeatureView (Ljava/lang/String;Ljava/lang/String;)V @@ -73,6 +74,7 @@ public final class com/posthog/PostHog$Companion : com/posthog/PostHogInterface public fun addExceptionStep (Ljava/lang/String;Ljava/util/Map;)V public fun alias (Ljava/lang/String;Ljava/util/Map;)V public fun capture (Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;)V + public fun captureAi (Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;)Ljava/util/UUID; public fun captureException (Ljava/lang/Throwable;Ljava/util/Map;Ljava/util/Map;)V public fun captureFeatureInteraction (Ljava/lang/String;Ljava/lang/String;)V public fun captureFeatureView (Ljava/lang/String;Ljava/lang/String;)V @@ -147,6 +149,9 @@ public final class com/posthog/PostHogCompression : java/lang/Enum { public class com/posthog/PostHogConfig { public static final field Companion Lcom/posthog/PostHogConfig$Companion; + public static final field DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES I + public static final field DEFAULT_CAPTURE_AI_MAX_QUEUE_SIZE I + public static final field DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS I public static final field DEFAULT_EU_ASSETS_HOST Ljava/lang/String; public static final field DEFAULT_EU_HOST Ljava/lang/String; public static final field DEFAULT_FEATURE_FLAG_CALLED_CACHE_SIZE I @@ -161,9 +166,14 @@ public class com/posthog/PostHogConfig { public synthetic fun (Ljava/lang/String;Ljava/lang/String;ZZZIZLjava/util/List;ZIIIIIILcom/posthog/PostHogEncryption;Lcom/posthog/PostHogOnFeatureFlags;ZLkotlin/jvm/functions/Function1;ZLcom/posthog/PersonProfiles;ZLjava/net/Proxy;Lcom/posthog/surveys/PostHogSurveysConfig;Lcom/posthog/logs/PostHogLogsConfig;Lkotlin/jvm/functions/Function6;Lkotlin/jvm/functions/Function5;Lcom/posthog/errortracking/PostHogErrorTrackingConfig;Ljava/lang/String;Lcom/posthog/PostHogBootstrapConfig;Lkotlin/jvm/functions/Function3;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun addBeforeSend (Lcom/posthog/PostHogBeforeSend;)V public final fun addIntegration (Lcom/posthog/PostHogIntegration;)V + public final fun getAiStoragePrefix ()Ljava/lang/String; public final fun getBeforeSendList ()Ljava/util/List; public final fun getBootstrap ()Lcom/posthog/PostHogBootstrapConfig; public final fun getCachePreferences ()Lcom/posthog/internal/PostHogPreferences; + public final fun getCaptureAiCompression ()Lcom/posthog/PostHogCompression; + public final fun getCaptureAiMaxEventBytes ()I + public final fun getCaptureAiMaxQueueSize ()I + public final fun getCaptureAiTimeout ()I public final fun getCompression ()Lcom/posthog/PostHogCompression; public final fun getContext ()Lcom/posthog/internal/PostHogContext; public final fun getDateProvider ()Lcom/posthog/internal/PostHogDateProvider; @@ -216,8 +226,13 @@ public class com/posthog/PostHogConfig { public final synthetic fun getTracingHeaders ()Ljava/util/List; public final fun removeBeforeSend (Lcom/posthog/PostHogBeforeSend;)V public final fun removeIntegration (Lcom/posthog/PostHogIntegration;)V + public final fun setAiStoragePrefix (Ljava/lang/String;)V public final fun setBootstrap (Lcom/posthog/PostHogBootstrapConfig;)V public final fun setCachePreferences (Lcom/posthog/internal/PostHogPreferences;)V + public final fun setCaptureAiCompression (Lcom/posthog/PostHogCompression;)V + public final fun setCaptureAiMaxEventBytes (I)V + public final fun setCaptureAiMaxQueueSize (I)V + public final fun setCaptureAiTimeout (I)V public final fun setCompression (Lcom/posthog/PostHogCompression;)V public final fun setContext (Lcom/posthog/internal/PostHogContext;)V public final fun setDateProvider (Lcom/posthog/internal/PostHogDateProvider;)V @@ -368,6 +383,8 @@ public abstract interface class com/posthog/PostHogInterface : com/posthog/PostH public static synthetic fun alias$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/util/Map;ILjava/lang/Object;)V public abstract fun capture (Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;)V public static synthetic fun capture$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;ILjava/lang/Object;)V + public abstract fun captureAi (Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;)Ljava/util/UUID; + public static synthetic fun captureAi$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;ILjava/lang/Object;)Ljava/util/UUID; public abstract fun captureException (Ljava/lang/Throwable;Ljava/util/Map;Ljava/util/Map;)V public static synthetic fun captureException$default (Lcom/posthog/PostHogInterface;Ljava/lang/Throwable;Ljava/util/Map;Ljava/util/Map;ILjava/lang/Object;)V public abstract fun captureFeatureInteraction (Ljava/lang/String;Ljava/lang/String;)V @@ -427,6 +444,7 @@ public final class com/posthog/PostHogInterface$DefaultImpls { public static synthetic fun addExceptionStep$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/util/Map;ILjava/lang/Object;)V public static synthetic fun alias$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/util/Map;ILjava/lang/Object;)V public static synthetic fun capture$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;ILjava/lang/Object;)V + public static synthetic fun captureAi$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Map;Ljava/util/Date;Ljava/util/Map;ILjava/lang/Object;)Ljava/util/UUID; public static synthetic fun captureException$default (Lcom/posthog/PostHogInterface;Ljava/lang/Throwable;Ljava/util/Map;Ljava/util/Map;ILjava/lang/Object;)V public static synthetic fun captureFeatureInteraction$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V public static synthetic fun captureFeatureView$default (Lcom/posthog/PostHogInterface;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)V @@ -615,12 +633,14 @@ public final class com/posthog/errortracking/PostHogExceptionStepsConfig { public final class com/posthog/internal/EndpointSpec { public static final field Companion Lcom/posthog/internal/EndpointSpec$Companion; + public static final fun ai (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; public static final fun batch (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; public static final fun logs (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; public static final fun snapshot (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; } public final class com/posthog/internal/EndpointSpec$Companion { + public final fun ai (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; public final fun batch (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; public final fun logs (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; public final fun snapshot (Lcom/posthog/PostHogConfig;Lcom/posthog/internal/PostHogApi;Ljava/lang/String;)Lcom/posthog/internal/EndpointSpec; @@ -735,6 +755,8 @@ public final class com/posthog/internal/FlagProperty { public final class com/posthog/internal/GzipRequestInterceptor : okhttp3/Interceptor { public fun (Lcom/posthog/PostHogConfig;)V + public fun (Lcom/posthog/PostHogConfig;Lkotlin/jvm/functions/Function0;)V + public synthetic fun (Lcom/posthog/PostHogConfig;Lkotlin/jvm/functions/Function0;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public fun intercept (Lokhttp3/Interceptor$Chain;)Lokhttp3/Response; } diff --git a/posthog/src/main/java/com/posthog/PostHog.kt b/posthog/src/main/java/com/posthog/PostHog.kt index a36d2b480..0cb5db421 100644 --- a/posthog/src/main/java/com/posthog/PostHog.kt +++ b/posthog/src/main/java/com/posthog/PostHog.kt @@ -94,6 +94,10 @@ public class PostHog private constructor( Executors.newSingleThreadScheduledExecutor( PostHogThreadFactory("PostHogLogsQueueThread"), ), + private val aiExecutor: ExecutorService = + Executors.newSingleThreadScheduledExecutor( + PostHogThreadFactory("PostHogAiQueueThread"), + ), private val remoteConfigExecutor: ExecutorService = Executors.newSingleThreadScheduledExecutor( PostHogThreadFactory("PostHogRemoteConfigThread"), @@ -126,6 +130,10 @@ public class PostHog private constructor( private var logsQueue: PostHogQueueInterface? = null + // The AI lane never shares a queue, executor or retry state with analytics events. + @Volatile + private var aiQueue: PostHogQueueInterface? = null + private var pushSubscriptionManager: PostHogPushSubscriptionManager? = null /** @@ -260,6 +268,12 @@ public class PostHog private constructor( EndpointSpec.logs(config, api, config.logsStoragePrefix), logsExecutor, ) + val aiQueue = + PostHogQueue( + config, + EndpointSpec.ai(config, api, config.aiStoragePrefix), + aiExecutor, + ) val onRemoteConfigLoaded = PostHogOnRemoteConfigLoaded { // The callback fires whether the attempt succeeded or terminally failed; @@ -329,6 +343,7 @@ public class PostHog private constructor( this.queue = queue this.replayQueue = replayQueue this.logsQueue = logsQueue + this.aiQueue = aiQueue this.pushSubscriptionManager = PostHogPushSubscriptionManager( config, @@ -372,6 +387,7 @@ public class PostHog private constructor( queue.start() logsQueue.start() + aiQueue.start() // Force the persisted OPT_OUT to hydrate before this passive push trigger: the // manager reads config.optOut directly, and it is otherwise resolved lazily on @@ -631,6 +647,7 @@ public class PostHog private constructor( queue?.stop() replayQueue?.stop() logsQueue?.stop() + aiQueue?.stop() pushSubscriptionManager?.close() pushSubscriptionManager = null @@ -999,16 +1016,70 @@ public class PostHog private constructor( timestamp: Date?, options: Map?, ) { + captureInternal( + event, + distinctId, + properties, + userProperties, + userPropertiesSetOnce, + groups, + timestamp, + options, + aiLane = false, + ) + } + + public override fun captureAi( + event: String, + distinctId: String?, + properties: Map?, + userProperties: Map?, + userPropertiesSetOnce: Map?, + groups: Map?, + timestamp: Date?, + options: Map?, + ): UUID? { + if (!event.startsWith("\$ai_")) { + config?.logger?.log("captureAi event $event doesn't start with \$ai_, so PostHog will drop it. Use capture for non-AI events.") + } + return captureInternal( + event, + distinctId, + properties, + userProperties, + userPropertiesSetOnce, + groups, + timestamp, + options, + aiLane = true, + ) + } + + /** + * Enriches and queues one event. Returns its uuid once it's handed to the queue, or null when + * it isn't. [aiLane] sends it through the AI queue instead of the analytics (or replay) queue. + */ + private fun captureInternal( + event: String, + distinctId: String?, + properties: Map?, + userProperties: Map?, + userPropertiesSetOnce: Map?, + groups: Map?, + timestamp: Date?, + options: Map?, + aiLane: Boolean, + ): UUID? { // Tracked on this call rather than derived from the event, so it survives beforeSend renaming // or replacing the event. var claimedReplayDebugBundle = false try { if (!isEnabled()) { - return + return null } if (isOptedOut()) { config?.logger?.log("PostHog is in OptOut state.") - return + return null } // A blank distinct ID falls back to the current one, like null, instead of PostHog dropping the event. @@ -1025,10 +1096,11 @@ public class PostHog private constructor( if (newDistinctId.isBlank()) { config?.logger?.log("capture call not allowed, distinctId is invalid: $newDistinctId.") - return + return null } - var isSnapshotEvent = event == PostHogEventName.SNAPSHOT.event + // The AI lane always uses its own queue. + var isSnapshotEvent = !aiLane && event == PostHogEventName.SNAPSHOT.event var groupIdentify = false if (event == PostHogEventName.GROUP_IDENTIFY.event) { groupIdentify = true @@ -1044,7 +1116,7 @@ public class PostHog private constructor( config?.logger?.log( "Skipping \$exception: an entry in \$exception_list matches ignoredExceptionTypes", ) - return + return null } // Attach the buffered exception steps to any $exception event (unless the caller @@ -1095,36 +1167,40 @@ public class PostHog private constructor( originalMessage } config?.logger?.log(message) - return + return null } // Reevaluate if this is a snapshot event because the event might have been updated by the beforeSend hook - isSnapshotEvent = postHogEvent.event == PostHogEventName.SNAPSHOT.event + isSnapshotEvent = !aiLane && postHogEvent.event == PostHogEventName.SNAPSHOT.event // if this is a $snapshot event and $session_id is missing, don't process then event if (isSnapshotEvent && postHogEvent.properties?.get("\$session_id") == null) { config?.logger?.log("${postHogEvent.event} event dropped, because the \$session_id property is missing") - return + return null } // Replay has its own queue if (isSnapshotEvent) { replayQueue?.add(postHogEvent) - } else { - val queue = queue - // beforeSend may have opted out - if (isOptedOut()) { - config?.logger?.log("PostHog is in OptOut state.") - } else if (queue != null) { - queue.add(postHogEvent) - if (claimedReplayDebugBundle) { - commitReplayDebugPropertiesWindow() - } + return uuid + } + val targetQueue = if (aiLane) aiQueue ?: return null else queue + var queued = false + // beforeSend may have opted out + if (isOptedOut()) { + config?.logger?.log("PostHog is in OptOut state.") + } else if (targetQueue != null) { + targetQueue.add(postHogEvent) + queued = true + if (claimedReplayDebugBundle) { + commitReplayDebugPropertiesWindow() } - // Notify surveys integration about the event - surveysHandler?.onEvent(event, mergedProperties) - // Notify session replay handler about the event for event triggers - sessionReplayHandler?.onEvent(event, mergedProperties) } + // Notify surveys integration about the event + surveysHandler?.onEvent(event, mergedProperties) + // Notify session replay handler about the event for event triggers + sessionReplayHandler?.onEvent(event, mergedProperties) + return if (queued) uuid else null } catch (e: Throwable) { config?.logger?.log("Capture failed: $e.") + return null } finally { if (claimedReplayDebugBundle) { releaseReplayDebugPropertiesClaim() @@ -2191,6 +2267,7 @@ public class PostHog private constructor( super.flush() replayQueue?.flush() logsQueue?.flush() + aiQueue?.flush() // See the setup() call site: hydrate opt-out before the manager reads the raw config field. isOptedOut() pushSubscriptionManager?.retryPending() @@ -2713,12 +2790,17 @@ public class PostHog private constructor( Executors.newSingleThreadScheduledExecutor( PostHogThreadFactory("PostHogLogsQueueThread"), ), + aiExecutor: ExecutorService = + Executors.newSingleThreadScheduledExecutor( + PostHogThreadFactory("PostHogAiQueueThread"), + ), ): PostHogInterface { val instance = PostHog( queueExecutor = queueExecutor, replayExecutor = replayExecutor, logsExecutor = logsExecutor, + aiExecutor = aiExecutor, remoteConfigExecutor = featureFlagsExecutor, cachedEventsExecutor = cachedEventsExecutor, reloadFeatureFlags = reloadFeatureFlags, @@ -2757,6 +2839,27 @@ public class PostHog private constructor( ) } + public override fun captureAi( + event: String, + distinctId: String?, + properties: Map?, + userProperties: Map?, + userPropertiesSetOnce: Map?, + groups: Map?, + timestamp: Date?, + options: Map?, + ): UUID? = + shared.captureAi( + event, + distinctId = distinctId, + properties = properties, + userProperties = userProperties, + userPropertiesSetOnce = userPropertiesSetOnce, + groups = groups, + timestamp = timestamp, + options = options, + ) + public override fun captureException( throwable: Throwable, properties: Map?, diff --git a/posthog/src/main/java/com/posthog/PostHogConfig.kt b/posthog/src/main/java/com/posthog/PostHogConfig.kt index da0268a16..9550b5c89 100644 --- a/posthog/src/main/java/com/posthog/PostHogConfig.kt +++ b/posthog/src/main/java/com/posthog/PostHogConfig.kt @@ -421,6 +421,49 @@ public open class PostHogConfig( */ public var compression: PostHogCompression = PostHogCompression.GZIP + /** + * How the SDK compresses [PostHog.captureAi] request bodies, separately from [compression]. + * + * When [httpClient] is set, it applies only if that client installs a [com.posthog.internal.GzipRequestInterceptor], + * which the SDK then reads this setting through for AI requests. + * + * Default: [PostHogCompression.GZIP]. + */ + public var captureAiCompression: PostHogCompression = PostHogCompression.GZIP + + /** + * Maximum number of [PostHog.captureAi] events kept in their on-disk queue before the oldest + * are dropped. + * + * AI events can be megabytes, so this is lower than [maxQueueSize]. The queue also keeps at most + * 50 MiB of AI events, whatever this is set to. A value of 0 or less uses the default. + * + * Default: 100. + */ + public var captureAiMaxQueueSize: Int = DEFAULT_CAPTURE_AI_MAX_QUEUE_SIZE + + /** + * Timeout in seconds of a [PostHog.captureAi] request: its HTTP client's call, read and write + * timeouts. + * + * AI requests can carry several MB, so this is longer than for other requests, which keep + * OkHttp's 10 second defaults. A value of 0 or less uses the default. + * + * Default: 60. + */ + public var captureAiTimeout: Int = DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS + + /** + * Maximum size, in bytes, of a [PostHog.captureAi] event. The SDK drops a larger event when it + * queues it, logging only the event name and size. + * + * Use it to lower the limit. PostHog drops an AI event whose properties are over 8 MiB, so a + * larger value, or a value of 0 or less, uses 8 MiB. + * + * Default: 8388608 (8 MiB). + */ + public var captureAiMaxEventBytes: Int = DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES + /** * Your PostHog project token, trimmed of leading and trailing whitespace. * @@ -475,6 +518,9 @@ public open class PostHogConfig( @PostHogInternal public var logsStoragePrefix: String? = null + @PostHogInternal + public var aiStoragePrefix: String? = null + @PostHogInternal public var cachePreferences: PostHogPreferences? = null @@ -589,6 +635,15 @@ public open class PostHogConfig( /** Shared default for [flushIntervalSeconds] and [PostHogLogsConfig.flushIntervalSeconds]. */ public const val DEFAULT_FLUSH_INTERVAL_SECONDS: Int = 30 + /** Default for [captureAiMaxQueueSize]. */ + public const val DEFAULT_CAPTURE_AI_MAX_QUEUE_SIZE: Int = 100 + + /** Default for [captureAiTimeout]. */ + public const val DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS: Int = 60 + + /** Default for [captureAiMaxEventBytes]: PostHog's 8 MiB limit on an AI event's properties. */ + public const val DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES: Int = 8 * 1024 * 1024 + public const val DEFAULT_US_HOST: String = "https://us.i.posthog.com" public const val DEFAULT_US_ASSETS_HOST: String = "https://us-assets.i.posthog.com" diff --git a/posthog/src/main/java/com/posthog/PostHogInterface.kt b/posthog/src/main/java/com/posthog/PostHogInterface.kt index 15476fc54..58e57dcef 100644 --- a/posthog/src/main/java/com/posthog/PostHogInterface.kt +++ b/posthog/src/main/java/com/posthog/PostHogInterface.kt @@ -103,6 +103,52 @@ public interface PostHogInterface : PostHogCoreInterface { options: Map? = null, ) + /** + * Captures an AI observability event, such as `$ai_generation`, `$ai_span`, `$ai_trace` or + * `$ai_embedding`, and sends it to PostHog's AI endpoint (`/i/v1/ai/events`), which accepts much + * larger events than [capture]. + * + * Takes the same arguments as [capture], merges properties in the same order and goes through + * the same `beforeSend` hooks. The event is queued on disk separately from other events and sent + * unchanged: the SDK doesn't redact or truncate prompts, outputs or media. PostHog drops an event + * sent here whose name doesn't start with `$ai_`, and the SDK logs a debug message when it's + * given one. [capture] never sends events here, even ones named `$ai_*`. + * + * AI events use [PostHogConfig.flushAt], [PostHogConfig.maxBatchSize] and + * [PostHogConfig.flushIntervalSeconds], like other events, and + * [PostHogConfig.captureAiCompression], [PostHogConfig.captureAiMaxQueueSize] and + * [PostHogConfig.captureAiTimeout] instead of `compression`, `maxQueueSize` and the default + * timeouts. An event over [PostHogConfig.captureAiMaxEventBytes] (8 MiB by default) is dropped + * when it's queued, with a log line naming the event and its size. Requests carry up to about + * 5 MiB of events, and the SDK sends as soon as that much is queued. Up to 50 MiB of AI events + * are kept on disk; beyond that, the oldest are dropped. + * + * Example: + * ``` + * PostHog.captureAi("$ai_generation", properties = mapOf("$ai_model" to "gpt-5", "$ai_trace_id" to traceId)) + * ``` + * @param event the AI event name, such as `$ai_generation` + * @param distinctId the distinctId, the generated [distinctId] is used if not given or blank + * @param properties the custom properties + * @param userProperties the user properties, set as a "$set" property + * @param userPropertiesSetOnce the user properties to set only once, set as a "$set_once" property + * @param groups the groups, set as a "$groups" property + * @param timestamp the event timestamp override. UTC is preferred; the equivalent instant is serialized in UTC. If omitted, the current time is used + * @param options settings that tell PostHog how to process this event, see `options` in [capture] + * @return the event's uuid once it's handed to the queue, or null when it isn't: the SDK is + * disabled or opted out, or `beforeSend` dropped it + */ + public fun captureAi( + event: String, + distinctId: String? = null, + properties: Map? = null, + userProperties: Map? = null, + userPropertiesSetOnce: Map? = null, + groups: Map? = null, + timestamp: Date? = null, + options: Map? = null, + ): UUID? + /** * Captures exceptions * @param throwable the Throwable error diff --git a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt index 5ee8d7f99..f9cbda7b0 100644 --- a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt +++ b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt @@ -18,6 +18,14 @@ import java.io.OutputStream * Endpoints without per-record results return an empty result and leave * [recordId] null. * + * The byte limits are measured on the stored record files. [maxBatchBytes] is + * a soft limit on one request: a record that doesn't fit starts the next + * request, a record larger than the limit goes alone, and the queue flushes as + * soon as that many bytes are queued. [maxQueueBytes] is a FIFO limit on the + * queued files on top of [maxQueueSize]; the newest record is always kept. + * [maxRecordBytes] drops a larger record when it is queued. Null means no + * limit. + * * The queue itself stays record-type-agnostic; everything that varies * between endpoints lives here. */ @@ -37,6 +45,9 @@ public class EndpointSpec internal constructor( internal val isFatalRecord: (Record) -> Boolean = { false }, internal val canBatchTogether: ((Record, Record) -> Boolean)? = null, internal val recordId: ((Record) -> String?)? = null, + internal val maxBatchBytes: Long? = null, + internal val maxQueueBytes: Long? = null, + internal val maxRecordBytes: Long? = null, ) { public companion object { @JvmStatic @@ -63,6 +74,42 @@ public class EndpointSpec internal constructor( isRetriableStatusCode = ::isCaptureV1RetriableStatusCode, isFatalRecord = { it.isFatalExceptionEvent() }, recordId = { event -> event.uuid?.toString() }, + maxBatchBytes = CAPTURE_V1_MAX_BATCH_BYTES, + ) + + /** + * Returns the [EndpointSpec] for the capture v1 AI endpoint (`/i/v1/ai/events`), used by + * `captureAi`. Same codec, results and retry policy as [batch], on its own queue: AI events + * can be megabytes, so the queue is capped by bytes as well as count, and an event over + * [PostHogConfig.captureAiMaxEventBytes] (plus the stored envelope) is dropped when queued. + */ + @JvmStatic + public fun ai( + config: PostHogConfig, + api: PostHogApi, + storagePrefix: String?, + ): EndpointSpec = + EndpointSpec( + recordsLabel = "AI events", + storagePrefix = storagePrefix, + initialCap = { it.maxBatchSize }, + initialFlushAt = { it.flushAt }, + maxQueueSize = { it.effectiveCaptureAiMaxQueueSize }, + flushIntervalSeconds = { it.flushIntervalSeconds }, + encode = { event, stream -> + config.serializer.serialize(event, stream.writer().buffered()) + }, + decode = { stream -> + config.serializer.deserialize(stream.reader().buffered()) + }, + // Name only: AI properties can hold prompts and media. + describe = { event -> "AI event ${event.event}" }, + send = { events -> api.captureAi(events) }, + isRetriableStatusCode = ::isCaptureV1RetriableStatusCode, + recordId = { event -> event.uuid?.toString() }, + maxBatchBytes = CAPTURE_V1_MAX_BATCH_BYTES, + maxQueueBytes = AI_MAX_QUEUE_BYTES, + maxRecordBytes = config.captureAiMaxStoredEventBytes, ) @JvmStatic @@ -155,6 +202,37 @@ internal class SendResult( */ internal fun isCaptureV1RetriableStatusCode(code: Int): Boolean = code == 408 || code in setOf(500, 502, 503, 504) +/** Soft request size for the capture v1 analytics and AI endpoints, well under their body limits. */ +internal const val CAPTURE_V1_MAX_BATCH_BYTES: Long = 5L * 1024 * 1024 + +/** Disk budget for queued AI events; the oldest are dropped first. */ +internal const val AI_MAX_QUEUE_BYTES: Long = 50L * 1024 * 1024 + +/** + * The stored AI event also holds its envelope (name, ids, timestamp), so this much is allowed on + * top of the properties limit, which PostHog measures on the properties alone. + */ +internal const val AI_EVENT_ENVELOPE_BYTES: Long = 64L * 1024 + +/** [PostHogConfig.captureAiMaxQueueSize], or its default when it isn't positive. */ +internal val PostHogConfig.effectiveCaptureAiMaxQueueSize: Int + get() = if (captureAiMaxQueueSize > 0) captureAiMaxQueueSize else PostHogConfig.DEFAULT_CAPTURE_AI_MAX_QUEUE_SIZE + +/** [PostHogConfig.captureAiTimeout], or its default when it isn't positive. */ +internal val PostHogConfig.effectiveCaptureAiTimeoutSeconds: Int + get() = if (captureAiTimeout > 0) captureAiTimeout else PostHogConfig.DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS + +/** + * Stored-size limit for an AI event: [PostHogConfig.captureAiMaxEventBytes], capped at PostHog's + * 8 MiB limit, plus the envelope allowance. + */ +internal val PostHogConfig.captureAiMaxStoredEventBytes: Long + get() { + val limit = PostHogConfig.DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES + val bytes = if (captureAiMaxEventBytes > 0) minOf(captureAiMaxEventBytes, limit) else limit + return bytes + AI_EVENT_ENVELOPE_BYTES + } + /** `/snapshot` (replay) retry policy: 408, 429, the listed 5xx, plus 3xx redirects. */ internal fun isSnapshotRetriableStatusCode(code: Int): Boolean = code in setOf(408, 429, 500, 502, 503, 504) || code in 300..399 diff --git a/posthog/src/main/java/com/posthog/internal/GzipRequestInterceptor.kt b/posthog/src/main/java/com/posthog/internal/GzipRequestInterceptor.kt index a84cdff8d..4c47a0022 100644 --- a/posthog/src/main/java/com/posthog/internal/GzipRequestInterceptor.kt +++ b/posthog/src/main/java/com/posthog/internal/GzipRequestInterceptor.kt @@ -37,74 +37,81 @@ import java.io.IOException /** * This interceptor compresses the HTTP request body. Many webservers can't handle this! * @property config The Config + * @property compression The setting that decides whether to compress, [PostHogConfig.compression] + * by default; the AI capture client reads [PostHogConfig.captureAiCompression] instead */ @PostHogInternal -public class GzipRequestInterceptor(private val config: PostHogConfig) : Interceptor { - @Throws(IOException::class) - override fun intercept(chain: Interceptor.Chain): Response { - val originalRequest = chain.request() - val body = originalRequest.body +public class GzipRequestInterceptor + @JvmOverloads + constructor( + private val config: PostHogConfig, + private val compression: () -> PostHogCompression = { config.compression }, + ) : Interceptor { + @Throws(IOException::class) + override fun intercept(chain: Interceptor.Chain): Response { + val originalRequest = chain.request() + val body = originalRequest.body - return if (config.compression != PostHogCompression.GZIP || - body == null || - originalRequest.header("Content-Encoding") != null || - body is MultipartBody - ) { - chain.proceed(originalRequest) - } else { - val compressedRequest = - try { - originalRequest.newBuilder() - .header("Content-Encoding", "gzip") - .method(originalRequest.method, forceContentLength(gzip(body))) - .build() - } catch (e: Throwable) { - config.logger.log("Failed to gzip the request body: $e.") + return if (compression() != PostHogCompression.GZIP || + body == null || + originalRequest.header("Content-Encoding") != null || + body is MultipartBody + ) { + chain.proceed(originalRequest) + } else { + val compressedRequest = + try { + originalRequest.newBuilder() + .header("Content-Encoding", "gzip") + .method(originalRequest.method, forceContentLength(gzip(body))) + .build() + } catch (e: Throwable) { + config.logger.log("Failed to gzip the request body: $e.") - originalRequest - } - chain.proceed(compressedRequest) + originalRequest + } + chain.proceed(compressedRequest) + } } - } - private fun gzip(body: RequestBody): RequestBody { - return object : RequestBody() { - override fun contentType(): MediaType? { - return body.contentType() - } + private fun gzip(body: RequestBody): RequestBody { + return object : RequestBody() { + override fun contentType(): MediaType? { + return body.contentType() + } - override fun contentLength(): Long { - return -1 // We don't know the compressed length in advance! - } + override fun contentLength(): Long { + return -1 // We don't know the compressed length in advance! + } - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - val gzipSink = GzipSink(sink).buffer() - body.writeTo(gzipSink) - gzipSink.close() + @Throws(IOException::class) + override fun writeTo(sink: BufferedSink) { + val gzipSink = GzipSink(sink).buffer() + body.writeTo(gzipSink) + gzipSink.close() + } } } - } - // https://github.com/square/okhttp/issues/350 - @Throws(IOException::class) - private fun forceContentLength(body: RequestBody): RequestBody { - val buffer = Buffer() - body.writeTo(buffer) + // https://github.com/square/okhttp/issues/350 + @Throws(IOException::class) + private fun forceContentLength(body: RequestBody): RequestBody { + val buffer = Buffer() + body.writeTo(buffer) - return object : RequestBody() { - override fun contentType(): MediaType? { - return body.contentType() - } + return object : RequestBody() { + override fun contentType(): MediaType? { + return body.contentType() + } - override fun contentLength(): Long { - return buffer.size - } + override fun contentLength(): Long { + return buffer.size + } - @Throws(IOException::class) - override fun writeTo(sink: BufferedSink) { - sink.write(buffer.snapshot()) + @Throws(IOException::class) + override fun writeTo(sink: BufferedSink) { + sink.write(buffer.snapshot()) + } } } } -} diff --git a/posthog/src/main/java/com/posthog/internal/PostHogApi.kt b/posthog/src/main/java/com/posthog/internal/PostHogApi.kt index 752dab9ee..4f52da0c2 100644 --- a/posthog/src/main/java/com/posthog/internal/PostHogApi.kt +++ b/posthog/src/main/java/com/posthog/internal/PostHogApi.kt @@ -48,6 +48,7 @@ public class PostHogApi( private const val FLAGS_MAX_RETRY_DELAY_MS = 30_000L private const val ERROR_BODY_PEEK_BYTES = 2048L private const val CAPTURE_V1_ANALYTICS_PATH = "/i/v1/analytics/events" + private const val CAPTURE_V1_AI_PATH = "/i/v1/ai/events" private const val CAPTURE_V1_MAX_REDIRECTS = 5 private const val CAPTURE_V1_MAX_RESULTS_BYTES = 1024L * 1024L } @@ -112,6 +113,24 @@ public class PostHogApi( .build() } + // AI batches can be several MB, so they get their own timeout and compression setting. + private val aiCaptureClient: OkHttpClient by lazy { + val timeoutSeconds = config.effectiveCaptureAiTimeoutSeconds.toLong() + val builder = + // Idle timeouts only: a multi-MB upload that is still making progress on a slow link + // must not be cut off by a whole-call deadline. + captureClient.newBuilder() + .readTimeout(timeoutSeconds, TimeUnit.SECONDS) + .writeTimeout(timeoutSeconds, TimeUnit.SECONDS) + val interceptors = builder.interceptors() + for (index in interceptors.indices) { + if (interceptors[index] is GzipRequestInterceptor) { + interceptors[index] = GzipRequestInterceptor(config) { config.captureAiCompression } + } + } + builder.build() + } + private val captureV1Lock = Any() // The identity of the last capture v1 request per endpoint path whose events are still to be @@ -155,8 +174,22 @@ public class PostHogApi( * Throws [PostHogApiError] for a non-2xx status and [IOException] for a transport failure. */ @Throws(PostHogApiError::class, IOException::class) - internal fun captureV1(events: List): SendResult { - val path = CAPTURE_V1_ANALYTICS_PATH + internal fun captureV1(events: List): SendResult = postCaptureV1(CAPTURE_V1_ANALYTICS_PATH, events, captureClient) + + /** + * Sends AI events to the capture v1 AI endpoint (`POST /i/v1/ai/events`). Same wire format, + * per-event results, redirect rule and retry identity (kept per path) as [captureV1], with + * [PostHogConfig.captureAiTimeout] and [PostHogConfig.captureAiCompression]. + */ + @Throws(PostHogApiError::class, IOException::class) + internal fun captureAi(events: List): SendResult = postCaptureV1(CAPTURE_V1_AI_PATH, events, aiCaptureClient) + + @Throws(PostHogApiError::class, IOException::class) + private fun postCaptureV1( + path: String, + events: List, + client: OkHttpClient, + ): SendResult { val uuids = events.mapNotNull { it.uuid?.toString() } val identity = nextCaptureV1Identity(path, uuids.toSet()) val body = @@ -168,7 +201,7 @@ public class PostHogApi( val url = "$theHost$path" var request = makeRequest(url, contentType = captureMediaType) { - // Events hold user data, so the debug log names only the count. + // Events hold user data, and AI events prompts and media, so the debug log names only the count. config.logger.log("Request $url: ${events.size} events") config.serializer.serialize(body, it.bufferedWriter()) @@ -184,7 +217,7 @@ public class PostHogApi( while (true) { logRequestHeaders(request) - captureClient.newCall(request).execute().use { response -> + client.newCall(request).execute().use { response -> // The status only: the results name event uuids. config.logger.log("Response ${response.request.url}: ${response.code}") diff --git a/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt b/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt index 830638b6c..8608c7330 100644 --- a/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt +++ b/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt @@ -35,6 +35,10 @@ public class PostHogQueue( // The files of the last request that failed or came back with records to retry. The next flush // resends exactly these, so the request keeps its id and counts the attempt. Guarded by dequeLock. private var retryFiles: List = emptyList() + + // Total size of the files in the deque, kept only for byte-limited endpoints. Guarded by dequeLock. + private val trackBytes = spec.maxBatchBytes != null || spec.maxQueueBytes != null + private var queuedBytes = 0L private val timerLock = Any() @PostHogVisibleForTesting @@ -99,6 +103,23 @@ public class PostHogQueue( os.use { theOutputStream -> spec.encode(record, theOutputStream) } + val size = file.length() + val maxRecordBytes = spec.maxRecordBytes + if (maxRecordBytes != null && size > maxRecordBytes) { + // Name and size only: the record can hold prompts and media. + config.logger.log("Dropping ${spec.describe(record)} of $size bytes, over the $maxRecordBytes byte limit.") + // Not counted in queuedBytes yet, so only the deque entry goes. + synchronized(dequeLock) { + deque.remove(file) + } + file.deleteSafely(config) + return false + } + if (trackBytes) { + synchronized(dequeLock) { + queuedBytes += size + } + } config.logger.log("Queued ${spec.describe(record)}: ${file.name}.") return true @@ -119,18 +140,45 @@ public class PostHogQueue( return true } + /** Removes [files] from the deque, subtracting their size while they still exist on disk. */ + private fun removeFromDeque(files: Collection) { + synchronized(dequeLock) { + deque.removeAll(files) + if (trackBytes) { + queuedBytes -= files.sumOf { it.length() } + } + } + } + private fun removeRecordSync() { if (deque.size > spec.maxQueueSize(config).coerceAtLeast(1)) { try { val first: File synchronized(dequeLock) { first = deque.removeFirst() + if (trackBytes) { + queuedBytes -= first.length() + } } first.deleteSafely(config) config.logger.log("Queue is full, the oldest ${spec.recordsLabel} ${first.name} is dropped.") } catch (ignored: NoSuchElementException) { } } + val maxQueueBytes = spec.maxQueueBytes ?: return + val evicted = mutableListOf() + synchronized(dequeLock) { + // The newest record is always kept, even when it's over the limit on its own. + while (queuedBytes > maxQueueBytes && deque.size > 1) { + val first = deque.removeFirst() + queuedBytes -= first.length() + evicted.add(first) + } + } + if (evicted.isNotEmpty()) { + evicted.forEach { it.deleteSafely(config) } + config.logger.log("Queue is over $maxQueueBytes bytes, the oldest ${evicted.size} ${spec.recordsLabel} are dropped.") + } } /** @@ -181,7 +229,7 @@ public class PostHogQueue( } private fun isAboveThreshold(flushAt: Int): Boolean { - if (deque.size >= flushAt) { + if (deque.size >= flushAt || isAboveByteThreshold()) { return true } else if (deque.size > 0) { // only log if there are records in the queue @@ -199,6 +247,18 @@ public class PostHogQueue( return true } + /** A full request's worth of queued bytes flushes like [BatchLimits.flushAt] does. */ + private fun isAboveByteThreshold(): Boolean { + val maxBatchBytes = spec.maxBatchBytes ?: return false + return synchronized(dequeLock) { queuedBytes } >= maxBatchBytes + } + + /** + * The next FIFO window: at most [BatchLimits.cap] files and, for a byte-limited endpoint, about + * [EndpointSpec.maxBatchBytes]. The check runs before each file is added, so a file that doesn't + * fit starts the next window and a file over the limit goes alone. The files of a request to + * retry go again on their own first. + */ private fun takeFiles(): List { synchronized(dequeLock) { val retry = retryFiles @@ -207,7 +267,17 @@ public class PostHogQueue( if (retry.isNotEmpty() && deque.containsAll(retry)) { return retry } - return deque.take(batchLimits.cap) + val maxBatchBytes = spec.maxBatchBytes ?: return deque.take(batchLimits.cap) + val files = mutableListOf() + var bytes = 0L + for (file in deque) { + if (files.size >= batchLimits.cap) break + val size = file.length() + if (files.isNotEmpty() && bytes + size > maxBatchBytes) break + files.add(file) + bytes += size + } + return files } } @@ -274,9 +344,7 @@ public class PostHogQueue( file: File, throwable: Throwable? = null, ) { - synchronized(dequeLock) { - deque.remove(file) - } + removeFromDeque(listOf(file)) file.deleteSafely(config) config.logger.log("File: ${file.name} failed to parse: $throwable.") } @@ -378,9 +446,7 @@ public class PostHogQueue( } finally { if (deleteFiles) { val delivered = files - retained.toSet() - synchronized(dequeLock) { - deque.removeAll(delivered) - } + removeFromDeque(delivered) delivered.forEach { it.deleteSafely(config) @@ -446,6 +512,7 @@ public class PostHogQueue( synchronized(dequeLock) { tempFiles = deque.toList() deque.clear() + queuedBytes = 0 } tempFiles.forEach { it.deleteSafely(config) @@ -517,6 +584,9 @@ public class PostHogQueue( deque.clear() deque.addAll(files) deque.addAll(existingFiles) + if (trackBytes) { + queuedBytes += files.sumOf { it.length() } + } } config.logger.log("Loaded ${files.size} cached ${spec.recordsLabel} from disk.") } @@ -537,7 +607,16 @@ public class PostHogQueue( files.sortBy { file -> file.lastModified() } val maxQueueSize = spec.maxQueueSize(config).coerceAtLeast(1) - val overflow = (files.size - maxQueueSize).coerceAtLeast(0) + var overflow = (files.size - maxQueueSize).coerceAtLeast(0) + val maxQueueBytes = spec.maxQueueBytes + if (maxQueueBytes != null) { + // Oldest first, keeping at least the newest file, as when a record is added. + var bytes = files.drop(overflow).sumOf { it.length() } + while (bytes > maxQueueBytes && overflow < files.size - 1) { + bytes -= files[overflow].length() + overflow++ + } + } if (overflow > 0) { files.take(overflow).forEach { it.deleteSafely(config) } config.logger.log("Dropped $overflow oldest cached ${spec.recordsLabel} to enforce queue capacity.") @@ -550,6 +629,9 @@ public class PostHogQueue( synchronized(dequeLock) { deque.clear() deque.addAll(files) + if (trackBytes) { + queuedBytes = files.sumOf { it.length() } + } } cachedRecordsLoaded = true } @@ -604,6 +686,10 @@ public class PostHogQueue( internal val currentRetryCountForTesting: Int @PostHogVisibleForTesting get() = retryCount + + internal val queuedBytesForTesting: Long + @PostHogVisibleForTesting + get() = synchronized(dequeLock) { queuedBytes } } internal class BatchLimits( diff --git a/posthog/src/testFixtures/java/com/posthog/PostHogFake.kt b/posthog/src/testFixtures/java/com/posthog/PostHogFake.kt index e7db1d47f..957123a20 100644 --- a/posthog/src/testFixtures/java/com/posthog/PostHogFake.kt +++ b/posthog/src/testFixtures/java/com/posthog/PostHogFake.kt @@ -51,6 +51,22 @@ public class PostHogFake : PostHogInterface { captures++ } + override fun captureAi( + event: String, + distinctId: String?, + properties: Map?, + userProperties: Map?, + userPropertiesSetOnce: Map?, + groups: Map?, + timestamp: Date?, + options: Map?, + ): UUID? { + this.event = event + this.properties = properties + captures++ + return UUID.randomUUID() + } + override fun captureLog( message: String, severity: PostHogLogSeverity, From 5c104bd97af13a13230b13044673b2e63b78492a Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Sat, 24 Oct 2026 13:57:09 -0400 Subject: [PATCH 02/19] test: cover captureAi, the AI lane and byte-aware batching SDK-level: routing by method, beforeSend and options, null returns, a terminal 404, the oversize drop (name and size only in the log), the non-`$ai_` debug line, flush and close draining the AI queue, compression per lane and the AI timeout. API-level: the AI endpoint's headers and body, separate retry identities per lane, and an AI queue keeping only the events asked to retry. Queue-level, on a synthetic record: byte-limited request windows, an oversize record sent alone, the flush at a request's worth of bytes, the per-record drop, the disk cap on add and on load, and the spec limits and config fallbacks. The Android setup test covers the AI storage prefix. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DRbQPidntd4hVufNpMT19L --- .../android/PostHogAndroidSetupLookupsTest.kt | 20 +- .../java/com/posthog/PostHogCaptureAiTest.kt | 292 ++++++++++++++++++ .../posthog/internal/PostHogCaptureV1Test.kt | 74 ++++- .../internal/PostHogQueueByteLimitsTest.kt | 229 ++++++++++++++ 4 files changed, 612 insertions(+), 3 deletions(-) create mode 100644 posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt create mode 100644 posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt diff --git a/posthog-android/src/test/java/com/posthog/android/PostHogAndroidSetupLookupsTest.kt b/posthog-android/src/test/java/com/posthog/android/PostHogAndroidSetupLookupsTest.kt index adf83051d..65c9f9c12 100644 --- a/posthog-android/src/test/java/com/posthog/android/PostHogAndroidSetupLookupsTest.kt +++ b/posthog-android/src/test/java/com/posthog/android/PostHogAndroidSetupLookupsTest.kt @@ -110,6 +110,7 @@ internal class PostHogAndroidSetupLookupsTest { assertEquals(File(cacheDir, "posthog-disk-queue").absolutePath, config.storagePrefix) assertEquals(File(cacheDir, "posthog-disk-replay-queue").absolutePath, config.replayStoragePrefix) assertEquals(File(cacheDir, "posthog-disk-logs-queue").absolutePath, config.logsStoragePrefix) + assertEquals(File(cacheDir, "posthog-disk-ai-queue").absolutePath, config.aiStoragePrefix) } @Test @@ -120,15 +121,29 @@ internal class PostHogAndroidSetupLookupsTest { storagePrefix = tmpDir.newFolder().absolutePath replayStoragePrefix = tmpDir.newFolder().absolutePath logsStoragePrefix = tmpDir.newFolder().absolutePath + aiStoragePrefix = tmpDir.newFolder().absolutePath } - val expected = listOf(config.legacyStoragePrefix, config.storagePrefix, config.replayStoragePrefix, config.logsStoragePrefix) + val expected = + listOf( + config.legacyStoragePrefix, + config.storagePrefix, + config.replayStoragePrefix, + config.logsStoragePrefix, + config.aiStoragePrefix, + ) setup(config) verify(app, never()).cacheDir verify(app, never()).getDir(any(), any()) assertEquals( expected, - listOf(config.legacyStoragePrefix, config.storagePrefix, config.replayStoragePrefix, config.logsStoragePrefix), + listOf( + config.legacyStoragePrefix, + config.storagePrefix, + config.replayStoragePrefix, + config.logsStoragePrefix, + config.aiStoragePrefix, + ), ) } @@ -142,6 +157,7 @@ internal class PostHogAndroidSetupLookupsTest { assertEquals(customPath, config.storagePrefix) assertEquals(File(cacheDir, "posthog-disk-replay-queue").absolutePath, config.replayStoragePrefix) assertEquals(File(cacheDir, "posthog-disk-logs-queue").absolutePath, config.logsStoragePrefix) + assertEquals(File(cacheDir, "posthog-disk-ai-queue").absolutePath, config.aiStoragePrefix) } @Test diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt new file mode 100644 index 000000000..476fdb62d --- /dev/null +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -0,0 +1,292 @@ +package com.posthog + +import com.posthog.internal.PostHogMemoryPreferences +import com.posthog.internal.PostHogThreadFactory +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okhttp3.mockwebserver.RecordedRequest +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +internal class PostHogCaptureAiTest { + @get:Rule + val tmpDir = TemporaryFolder() + + @get:Rule + val httpServers = TestHttpServers() + + private val queueExecutor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("TestQueue")) + private val replayQueueExecutor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("TestReplayQueue")) + private val logsExecutor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("TestLogsQueue")) + private val aiExecutor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("TestAiQueue")) + private val remoteConfigExecutor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("TestRemoteConfig")) + private val cachedEventsExecutor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("TestCachedEvents")) + private val logger = TestLogger() + private val clients = mutableListOf() + private lateinit var config: PostHogConfig + + private fun getSut( + http: MockWebServer, + configure: PostHogConfig.() -> Unit = {}, + ): PostHogInterface { + val storage = tmpDir.newFolder().absolutePath + config = + PostHogConfig(PROJECT_TOKEN, http.url("/").toString()).apply { + storagePrefix = File(storage, "events").absolutePath + replayStoragePrefix = File(storage, "snapshots").absolutePath + logsStoragePrefix = File(storage, "logs").absolutePath + aiStoragePrefix = File(storage, "ai").absolutePath + flushAt = 1 + preloadFeatureFlags = false + sendFeatureFlagEvent = false + cachePreferences = PostHogMemoryPreferences() + context = TestPostHogContext() + logger = this@PostHogCaptureAiTest.logger + configure() + } + return PostHog.withInternal( + config, + queueExecutor, + replayQueueExecutor, + remoteConfigExecutor, + cachedEventsExecutor, + reloadFeatureFlags = false, + logsExecutor = logsExecutor, + aiExecutor = aiExecutor, + ).also { clients.add(it) } + } + + @AfterTest + fun cleanup() { + clients.forEach { it.close() } + listOf(queueExecutor, replayQueueExecutor, logsExecutor, aiExecutor, remoteConfigExecutor, cachedEventsExecutor).forEach { + it.shutdownAndAwaitTermination() + } + tmpDir.root.deleteRecursively() + } + + private fun aiQueueFiles(): List = File(config.aiStoragePrefix!!, PROJECT_TOKEN).listFiles()?.toList() ?: emptyList() + + private fun takeRequest(http: MockWebServer): RecordedRequest = assertNotNull(http.takeRequest(5, TimeUnit.SECONDS)) + + private fun events(request: RecordedRequest): List> = captureV1Events(request.body.unGzip()) + + @Suppress("UNCHECKED_CAST") + private fun Map.properties(): Map = this["properties"] as Map + + @Test + fun `captureAi posts to the AI endpoint and returns the uuid on the wire`() { + val http = httpServers.mockHttp() + val sut = getSut(http) + + val uuid = sut.captureAi("\$ai_generation", properties = mapOf("\$ai_model" to "gpt-5")) + + aiExecutor.awaitExecution() + val request = takeRequest(http) + assertEquals("/i/v1/ai/events", request.path) + assertEquals("Bearer $PROJECT_TOKEN", request.headers["Authorization"]) + assertEquals("1", request.headers["PostHog-Attempt"]) + val event = events(request).single() + assertEquals("\$ai_generation", event["event"]) + assertEquals(assertNotNull(uuid).toString(), event["uuid"]) + assertEquals(sut.distinctId(), event["distinct_id"]) + // same enrichment as capture: SDK context and the caller's properties + assertEquals("gpt-5", event.properties()["\$ai_model"]) + assertEquals("Android", event.properties()["\$os_name"]) + queueExecutor.awaitExecution() + assertEquals(1, http.requestCount) + } + + @Test + fun `capture keeps an ai-named event on the analytics endpoint`() { + val http = httpServers.mockHttp() + val sut = getSut(http) + + sut.capture("\$ai_generation") + + queueExecutor.awaitExecution() + aiExecutor.awaitExecution() + assertEquals("/i/v1/analytics/events", takeRequest(http).path) + assertEquals(1, http.requestCount) + assertTrue(aiQueueFiles().isEmpty()) + } + + @Test + fun `beforeSend, options and the personless default apply to AI events`() { + val http = httpServers.mockHttp() + val sut = + getSut(http) { + addBeforeSend { event -> + event.properties?.remove("\$ai_input") + event.properties?.put("redacted", true) + event + } + } + + sut.captureAi( + "\$ai_generation", + properties = mapOf("\$ai_input" to "secret prompt"), + options = mapOf("cookieless_mode" to true), + ) + + aiExecutor.awaitExecution() + val event = events(takeRequest(http)).single() + assertEquals(true, event.properties()["redacted"]) + assertTrue("\$ai_input" !in event.properties()) + // identified-only without an identify: the SDK adds the personless option + assertEquals(mapOf("cookieless_mode" to true, "process_person_profile" to false), event["options"]) + } + + @Test + fun `captureAi returns null when disabled, opted out or rejected by beforeSend`() { + val http = httpServers.mockHttp() + val sut = getSut(http) { addBeforeSend { event -> if (event.event == "\$ai_rejected") null else event } } + + assertNull(sut.captureAi("\$ai_rejected")) + sut.optOut() + assertNull(sut.captureAi("\$ai_generation")) + sut.optIn() + sut.close() + assertNull(sut.captureAi("\$ai_generation")) + + aiExecutor.awaitExecution() + assertEquals(0, http.requestCount) + assertTrue(aiQueueFiles().isEmpty()) + } + + @Test + fun `a 404 from the AI endpoint is terminal`() { + val http = httpServers.mockHttp(total = 2, response = MockResponse().setResponseCode(404)) + val sut = getSut(http) + + sut.captureAi("\$ai_generation") + aiExecutor.awaitExecution() + sut.flush() + aiExecutor.awaitExecution() + + assertEquals(1, http.requestCount) + assertEquals("/i/v1/ai/events", takeRequest(http).path) + assertTrue(aiQueueFiles().isEmpty()) + } + + @Test + fun `an AI event over captureAiMaxEventBytes is dropped when queued and logs only its name and size`() { + val http = httpServers.mockHttp() + // the stored limit is the configured bytes plus a 64 KiB envelope + val sut = getSut(http) { captureAiMaxEventBytes = 1 } + val prompt = "p".repeat(70_000) + + val uuid = sut.captureAi("\$ai_generation", properties = mapOf("\$ai_input" to prompt)) + + // handed to the queue, which drops it on its own thread + assertNotNull(uuid) + aiExecutor.awaitExecution() + sut.flush() + aiExecutor.awaitExecution() + assertEquals(0, http.requestCount) + assertTrue(aiQueueFiles().isEmpty()) + val dropped = logger.messages.single { it.startsWith("Dropping AI event") } + assertTrue( + dropped.matches(Regex("Dropping AI event \\\$ai_generation of \\d+ bytes, over the ${1 + 64 * 1024} byte limit\\.")), + dropped, + ) + assertTrue(logger.messages.none { it.contains("pppp") }) + } + + @Test + fun `a name that doesn't start with ai is logged and sent unchanged`() { + val http = httpServers.mockHttp() + val sut = getSut(http) + + sut.captureAi("not_ai") + + aiExecutor.awaitExecution() + val request = takeRequest(http) + assertEquals("/i/v1/ai/events", request.path) + assertEquals("not_ai", events(request).single()["event"]) + assertTrue(logger.messages.any { it.startsWith("captureAi event not_ai doesn't start with \$ai_") }) + } + + @Test + fun `flush and close drain the AI queue`() { + val http = httpServers.mockHttp(total = 2) + val sut = getSut(http) { flushAt = 10 } + + sut.captureAi("\$ai_generation") + aiExecutor.awaitExecution() + assertEquals(0, http.requestCount) + + sut.flush() + aiExecutor.awaitExecution() + assertEquals(1, http.requestCount) + + sut.captureAi("\$ai_span") + aiExecutor.awaitExecution() + sut.close() + aiExecutor.awaitExecution() + assertEquals(2, http.requestCount) + assertEquals(listOf("/i/v1/ai/events", "/i/v1/ai/events"), List(2) { takeRequest(http).path }) + } + + @Test + fun `compression is set per lane`() { + val http = httpServers.mockHttp(total = 2) + val sut = + getSut(http) { + compression = PostHogCompression.NONE + captureAiCompression = PostHogCompression.GZIP + } + + sut.capture("event") + sut.captureAi("\$ai_generation") + + queueExecutor.awaitExecution() + aiExecutor.awaitExecution() + val requests = List(2) { takeRequest(http) }.associateBy { it.path } + val analytics = assertNotNull(requests["/i/v1/analytics/events"]) + val ai = assertNotNull(requests["/i/v1/ai/events"]) + assertNull(analytics.headers["Content-Encoding"]) + assertTrue(analytics.body.readUtf8().contains("\"event\":\"event\"")) + assertEquals("gzip", ai.headers["Content-Encoding"]) + assertEquals("\$ai_generation", events(ai).single()["event"]) + } + + @Test + fun `captureAiCompression off sends the AI request uncompressed while analytics stay gzipped`() { + val http = httpServers.mockHttp(total = 2) + val sut = getSut(http) { captureAiCompression = PostHogCompression.NONE } + + sut.capture("event") + sut.captureAi("\$ai_generation") + + queueExecutor.awaitExecution() + aiExecutor.awaitExecution() + val requests = List(2) { takeRequest(http) }.associateBy { it.path } + assertEquals("gzip", assertNotNull(requests["/i/v1/analytics/events"]).headers["Content-Encoding"]) + val ai = assertNotNull(requests["/i/v1/ai/events"]) + assertNull(ai.headers["Content-Encoding"]) + assertEquals("\$ai_generation", captureV1Events(ai.body.readUtf8()).single()["event"]) + } + + @Test + fun `AI requests time out after captureAiTimeout and keep the event for retry`() { + val http = httpServers.mockHttp(response = MockResponse().setHeadersDelay(3, TimeUnit.SECONDS)) + val sut = getSut(http) { captureAiTimeout = 1 } + + sut.captureAi("\$ai_generation") + aiExecutor.awaitExecution() + + assertEquals(1, http.requestCount) + assertEquals(1, aiQueueFiles().size) + assertTrue(logger.messages.any { it.contains("network error") }) + } +} diff --git a/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt b/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt index 492cec17b..2916d9eb6 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt @@ -66,6 +66,7 @@ internal class PostHogCaptureV1Test { http: MockWebServer, flushAt: Int = 1, storagePrefix: String = tmpDir.newFolder().absolutePath, + ai: Boolean = false, ): PostHogQueue { val config = config(http.url("/").toString()).apply { @@ -74,7 +75,8 @@ internal class PostHogCaptureV1Test { } // pause time pinned to the past so a backoff never blocks the next flush clock.setAddSecondsToCurrentDate(parseISO8601Date("1970-01-01T00:00:00.000Z")!!) - val spec = EndpointSpec.batch(config, PostHogApi(config), config.storagePrefix) + val api = PostHogApi(config) + val spec = if (ai) EndpointSpec.ai(config, api, config.storagePrefix) else EndpointSpec.batch(config, api, config.storagePrefix) return PostHogQueue(config, spec, executor).also { queues.add(it) } } @@ -132,6 +134,57 @@ internal class PostHogCaptureV1Test { assertEquals("gzip", request.headers["Content-Encoding"]) } + @Test + fun `captureAi posts to the AI endpoint with the same headers and body shape`() { + val http = httpServers.mockHttp() + clock.setCurrentDate(parseISO8601Date("2024-05-01T10:20:30.456Z")!!) + val event = event("\$ai_generation") + + api(http.url("/").toString()).captureAi(listOf(event)) + + val request = http.takeRequest() + assertEquals("POST", request.method) + assertEquals("/i/v1/ai/events", request.path) + assertEquals("Bearer $PROJECT_TOKEN", request.headers["Authorization"]) + assertEquals("posthog-java/${BuildConfig.VERSION_NAME}", request.headers["PostHog-Sdk-Info"]) + assertEquals("1", request.headers["PostHog-Attempt"]) + UUID.fromString(assertNotNull(request.headers["PostHog-Request-Id"])) + assertEquals("2024-05-01T10:20:30.456Z", request.headers["PostHog-Request-Timestamp"]) + assertEquals("application/json", request.headers["Content-Type"]) + assertEquals("gzip", request.headers["Content-Encoding"]) + val body = body(request) + assertEquals(setOf("created_at", "batch"), body.keys) + @Suppress("UNCHECKED_CAST") + assertEquals(listOf(event.uuid.toString()), (body["batch"] as List>).map { it["uuid"] }) + } + + @Test + fun `analytics and AI retries keep separate request identities`() { + val analytics = event() + val ai = event("\$ai_generation") + val http = httpServers.mockHttp(response = resultsResponse(analytics.uuid!! to "retry")) + http.enqueue(resultsResponse(ai.uuid!! to "retry")) + http.enqueue(resultsResponse(analytics.uuid!! to "ok")) + http.enqueue(resultsResponse(ai.uuid!! to "ok")) + val sut = api(http.url("/").toString()) + + sut.captureV1(listOf(analytics)) + sut.captureAi(listOf(ai)) + sut.captureV1(listOf(analytics)) + sut.captureAi(listOf(ai)) + + val requests = List(4) { http.takeRequest() } + assertEquals( + listOf("/i/v1/analytics/events", "/i/v1/ai/events", "/i/v1/analytics/events", "/i/v1/ai/events"), + requests.map { it.path }, + ) + assertEquals(listOf("1", "1", "2", "2"), requests.map { it.headers["PostHog-Attempt"] }) + val ids = requests.map { it.headers["PostHog-Request-Id"] } + assertNotEquals(ids[0], ids[1]) + assertEquals(ids[0], ids[2]) + assertEquals(ids[1], ids[3]) + } + @Test fun `the capture Bearer token replaces a custom Authorization header`() { val http = httpServers.mockHttp() @@ -610,6 +663,25 @@ internal class PostHogCaptureV1Test { assertEquals(1, sut.dequeList.size) } + @Test + fun `AI queue keeps only the events the AI endpoint asked to retry`() { + val ok = event("\$ai_generation") + val retry = event("\$ai_span") + val http = httpServers.mockHttp(response = resultsResponse(ok.uuid!! to "ok", retry.uuid!! to "retry")) + val sut = queue(http, flushAt = 2, ai = true) + + sut.add(ok) + sut.add(retry) + executor.awaitExecution() + + assertEquals("/i/v1/ai/events", http.takeRequest().path) + assertEquals(1, sut.currentRetryCountForTesting) + assertEquals( + listOf(retry.uuid.toString()), + sut.dequeList.map { file -> file.reader().use { serializer.deserialize(it).uuid.toString() } }, + ) + } + @Test fun `queue drops the batch on 429 and keeps it on 503`() { val http = httpServers.mockHttp(response = MockResponse().setResponseCode(429)) diff --git a/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt new file mode 100644 index 000000000..34c5dc58e --- /dev/null +++ b/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt @@ -0,0 +1,229 @@ +package com.posthog.internal + +import com.posthog.PROJECT_TOKEN +import com.posthog.PostHogConfig +import com.posthog.TestLogger +import com.posthog.awaitExecution +import com.posthog.shutdownAndAwaitTermination +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.UUID +import java.util.concurrent.Executors +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * The byte limits of [EndpointSpec] as [PostHogQueue] applies them, on a synthetic record whose + * stored size is its payload length. + */ +internal class PostHogQueueByteLimitsTest { + @get:Rule + val tmpDir = TemporaryFolder() + + private val executor = Executors.newSingleThreadScheduledExecutor(PostHogThreadFactory("Test")) + private val logger = TestLogger() + private val sent = mutableListOf>() + + @Volatile + private var connected = true + + private lateinit var config: PostHogConfig + private lateinit var storagePrefix: String + + @AfterTest + fun cleanup() { + executor.shutdownAndAwaitTermination() + } + + private fun record( + id: String, + bytes: Int, + ): String = id.padEnd(bytes, 'x') + + private fun queue( + maxBatchBytes: Long? = null, + maxQueueBytes: Long? = null, + maxRecordBytes: Long? = null, + flushAt: Int = 50, + maxQueueSize: Int = 1000, + ): PostHogQueue { + storagePrefix = tmpDir.newFolder().absolutePath + config = + PostHogConfig(PROJECT_TOKEN).apply { + logger = this@PostHogQueueByteLimitsTest.logger + networkStatus = + object : PostHogNetworkStatus { + override fun isConnected(): Boolean = connected + } + } + val spec = + EndpointSpec( + recordsLabel = "blobs", + storagePrefix = storagePrefix, + initialCap = { 50 }, + initialFlushAt = { flushAt }, + maxQueueSize = { maxQueueSize }, + flushIntervalSeconds = { 30 }, + encode = { record, stream -> stream.writer().use { it.write(record) } }, + decode = { stream -> stream.reader().use { it.readText() } }, + describe = { record -> "blob ${record.trimEnd('x')}" }, + send = { records -> + sent.add(records.map { it.trimEnd('x') }) + SendResult() + }, + isRetriableStatusCode = { false }, + maxBatchBytes = maxBatchBytes, + maxQueueBytes = maxQueueBytes, + maxRecordBytes = maxRecordBytes, + ) + return PostHogQueue(config, spec, executor) + } + + private fun queuedIds(sut: PostHogQueue): List = sut.dequeList.map { it.readText().trimEnd('x') } + + @Test + fun `a flush splits the queue into requests at the byte limit`() { + val sut = queue(maxBatchBytes = 100) + connected = false + sut.add(record("a", 40)) + sut.add(record("b", 40)) + sut.add(record("c", 40)) + executor.awaitExecution() + connected = true + + sut.flush() + executor.awaitExecution() + + assertEquals(listOf(listOf("a", "b"), listOf("c")), sent) + assertTrue(sut.dequeList.isEmpty()) + assertEquals(0L, sut.queuedBytesForTesting) + } + + @Test + fun `a record over the byte limit is sent alone and keeps its place`() { + val sut = queue(maxBatchBytes = 100) + connected = false + sut.add(record("a", 40)) + sut.add(record("big", 150)) + sut.add(record("c", 40)) + executor.awaitExecution() + connected = true + + sut.flush() + executor.awaitExecution() + + assertEquals(listOf(listOf("a"), listOf("big"), listOf("c")), sent) + } + + @Test + fun `flushes once a request's worth of bytes is queued, before flushAt`() { + val sut = queue(maxBatchBytes = 100, flushAt = 50) + + sut.add(record("a", 60)) + executor.awaitExecution() + assertTrue(sent.isEmpty()) + + sut.add(record("b", 60)) + executor.awaitExecution() + + // one request's worth: a alone, since b doesn't fit next to it + assertEquals(listOf(listOf("a")), sent) + assertEquals(listOf("b"), queuedIds(sut)) + } + + @Test + fun `drops a record over maxRecordBytes, logging its description and size`() { + val sut = queue(maxRecordBytes = 50, maxQueueBytes = 1000, flushAt = 50) + + sut.add(record("big", 80)) + sut.add(record("small", 20)) + executor.awaitExecution() + + assertEquals(listOf("small"), queuedIds(sut)) + assertEquals(1, File(storagePrefix, PROJECT_TOKEN).listFiles()!!.size) + // the dropped record was never counted, so it must not be subtracted either + assertEquals(sut.dequeList.single().length(), sut.queuedBytesForTesting) + assertEquals(listOf("Dropping blob big of 80 bytes, over the 50 byte limit."), logger.messages.filter { it.startsWith("Dropping") }) + } + + @Test + fun `evicts the oldest records over maxQueueBytes when a record is added`() { + val sut = queue(maxQueueBytes = 100, flushAt = 50) + + sut.add(record("a", 40)) + sut.add(record("b", 40)) + sut.add(record("c", 40)) + executor.awaitExecution() + assertEquals(listOf("b", "c"), queuedIds(sut)) + + // the newest record is kept even when it's over the limit on its own + sut.add(record("big", 150)) + executor.awaitExecution() + assertEquals(listOf("big"), queuedIds(sut)) + assertEquals(sut.dequeList.single().length(), sut.queuedBytesForTesting) + assertEquals(1, File(storagePrefix, PROJECT_TOKEN).listFiles()!!.size) + assertTrue(logger.messages.any { it == "Queue is over 100 bytes, the oldest 1 blobs are dropped." }) + } + + @Test + fun `evicts the oldest cached records over maxQueueBytes on load`() { + val sut = queue(maxQueueBytes = 100, flushAt = 50) + val dir = File(storagePrefix, PROJECT_TOKEN).apply { mkdirs() } + val cached = + listOf("a", "b", "c").mapIndexed { index, id -> + File(dir, "${UUID.randomUUID()}.event").apply { + writeText(record(id, 40)) + setLastModified(System.currentTimeMillis() - (3 - index) * 1000L) + } + } + + sut.reloadFromDisk() + + assertEquals(cached.drop(1), sut.dequeList) + assertEquals(listOf("b", "c"), queuedIds(sut)) + assertEquals(2, dir.listFiles()!!.size) + } + + @Test + fun `the analytics spec caps a request at 5 MiB and the AI spec adds the disk and event caps`() { + val config = PostHogConfig(PROJECT_TOKEN) + val api = PostHogApi(config) + + val batch = EndpointSpec.batch(config, api, null) + assertEquals(5L * 1024 * 1024, batch.maxBatchBytes) + assertEquals(null, batch.maxQueueBytes) + assertEquals(null, batch.maxRecordBytes) + + val ai = EndpointSpec.ai(config, api, null) + assertEquals(5L * 1024 * 1024, ai.maxBatchBytes) + assertEquals(50L * 1024 * 1024, ai.maxQueueBytes) + assertEquals(8L * 1024 * 1024 + 64 * 1024, ai.maxRecordBytes) + assertEquals(100, ai.maxQueueSize(config)) + } + + @Test + fun `the AI event limit can only be lowered and non-positive settings use the defaults`() { + val config = PostHogConfig(PROJECT_TOKEN) + val api = PostHogApi(config) + + config.captureAiMaxEventBytes = 1024 + assertEquals(1024L + 64 * 1024, EndpointSpec.ai(config, api, null).maxRecordBytes) + config.captureAiMaxEventBytes = 16 * 1024 * 1024 + assertEquals(8L * 1024 * 1024 + 64 * 1024, EndpointSpec.ai(config, api, null).maxRecordBytes) + config.captureAiMaxEventBytes = 0 + assertEquals(8L * 1024 * 1024 + 64 * 1024, EndpointSpec.ai(config, api, null).maxRecordBytes) + + config.captureAiMaxQueueSize = 0 + assertEquals(100, EndpointSpec.ai(config, api, null).maxQueueSize(config)) + config.captureAiMaxQueueSize = 7 + assertEquals(7, EndpointSpec.ai(config, api, null).maxQueueSize(config)) + + config.captureAiTimeout = -1 + assertEquals(60, config.effectiveCaptureAiTimeoutSeconds) + config.captureAiTimeout = 5 + assertEquals(5, config.effectiveCaptureAiTimeoutSeconds) + } +} From 51070c1a0b37e94f035ab97a8dd8d61c17fea206 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Sat, 24 Oct 2026 13:57:09 -0400 Subject: [PATCH 03/19] chore(compliance): run the capture v1 AI suite against the adapter Adds `POST /capture_ai`, which routes through `captureAi` and returns the uuid it sent, tracks `/i/v1/ai/events` requests next to the analytics ones and advertises `capture_ai_v1`. Also restores the capture v1 adapter changes (tracking by capture path, `PostHog-Attempt` as the retry count, `personProfiles = ALWAYS`, the renamed push retries field) that the options commit reverted. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DRbQPidntd4hVufNpMT19L --- .../IMPLEMENTATION_NOTES.md | 7 +- sdk_compliance_adapter/README.md | 6 +- .../posthog/compliance/ComplianceAdapter.kt | 69 +++++++++++++++++-- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/sdk_compliance_adapter/IMPLEMENTATION_NOTES.md b/sdk_compliance_adapter/IMPLEMENTATION_NOTES.md index a678f02e4..723403d76 100644 --- a/sdk_compliance_adapter/IMPLEMENTATION_NOTES.md +++ b/sdk_compliance_adapter/IMPLEMENTATION_NOTES.md @@ -76,11 +76,11 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for local build and compliance test comma ## SDK Type -The Android SDK advertises `capture_v1` and `encoding_gzip`: -- Endpoint: `POST /i/v1/analytics/events` +The Android SDK advertises `capture_v1`, `capture_ai_v1`, `encoding_gzip` and `event_options`: +- Endpoints: `POST /i/v1/analytics/events` (`/capture`) and `POST /i/v1/ai/events` (`/capture_ai`) - Format: `{created_at: "...", batch: [{event}, {event}]}` with the project token in an `Authorization: Bearer` header -Tests run with `--sdk-type server` flag; the harness selects the `capture_v1` suite from the capabilities. +Tests run with `--sdk-type server` flag; the harness selects the `capture_v1` and `capture_ai_v1` suites from the capabilities. ## API Endpoints @@ -89,6 +89,7 @@ The adapter implements the standard compliance adapter API: - `GET /health` - Health check with SDK version info - `POST /init` - Initialize SDK with config - `POST /capture` - Capture a single event +- `POST /capture_ai` - Capture a single AI event through `captureAi` - `POST /flush` - Force flush all pending events - `GET /state` - Get adapter state for assertions - `POST /reset` - Reset SDK and adapter state diff --git a/sdk_compliance_adapter/README.md b/sdk_compliance_adapter/README.md index c39b7e409..ca843b536 100644 --- a/sdk_compliance_adapter/README.md +++ b/sdk_compliance_adapter/README.md @@ -41,11 +41,11 @@ Events are tracked via: ### SDK Type -The Android SDK advertises `capture_v1` and `encoding_gzip`: -- Endpoint: `POST /i/v1/analytics/events` +The Android SDK advertises `capture_v1`, `capture_ai_v1`, `encoding_gzip` and `event_options`: +- Endpoints: `POST /i/v1/analytics/events` (`/capture`) and `POST /i/v1/ai/events` (`/capture_ai`) - Format: `{created_at, batch}` with the project token in an `Authorization: Bearer` header -Tests run with `--sdk-type server`; the harness selects the `capture_v1` suite from the capabilities. +Tests run with `--sdk-type server`; the harness selects the `capture_v1` and `capture_ai_v1` suites from the capabilities. ## Files Created diff --git a/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt b/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt index 46e217163..4e151ef6a 100644 --- a/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt +++ b/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt @@ -22,6 +22,9 @@ import io.ktor.server.routing.routing import okhttp3.Interceptor import okhttp3.Response import java.io.File +import java.time.OffsetDateTime +import java.time.format.DateTimeParseException +import java.util.Date import java.util.UUID import java.util.concurrent.TimeUnit import java.util.concurrent.locks.Condition @@ -34,8 +37,11 @@ private const val QUEUE_STORAGE_PREFIX = "/tmp/posthog-queue" // Upper bound on the wait for a flush()'d batch to reach the mock (generous for Docker). private const val FLUSH_SETTLE_MS = 2000L -// The capture endpoint the SDK posts analytics events to. +// The capture v1 endpoints the SDK posts analytics and AI events to. private const val CAPTURE_PATH = "/i/v1/analytics/events" +private const val CAPTURE_AI_PATH = "/i/v1/ai/events" + +private fun isCaptureUrl(url: String): Boolean = url.contains(CAPTURE_PATH) || url.contains(CAPTURE_AI_PATH) /** * PostHog Android SDK Compliance Adapter @@ -162,7 +168,7 @@ class TrackingInterceptor : Interceptor { val uuidList = mutableListOf() // Parse request body before sending - if (url.contains(CAPTURE_PATH) && requestBody != null) { + if (isCaptureUrl(url) && requestBody != null) { try { val buffer = okio.Buffer() requestBody.writeTo(buffer) @@ -183,7 +189,7 @@ class TrackingInterceptor : Interceptor { val response = chain.proceed(request) // Track the response - if (url.contains(CAPTURE_PATH)) { + if (isCaptureUrl(url)) { try { // PostHog-Attempt starts at 1; the harness counts retries from 0. val retryCount = (request.header("PostHog-Attempt")?.toIntOrNull() ?: 1) - 1 @@ -219,6 +225,16 @@ class TrackingInterceptor : Interceptor { } } +// The harness sends RFC 3339 timestamps, with any offset; the SDK serializes the instant in UTC. +private fun parseHarnessTimestamp(timestamp: String?): Date? = + timestamp?.let { + try { + Date.from(OffsetDateTime.parse(it).toInstant()) + } catch (_: DateTimeParseException) { + null + } + } + // Block until [predicate] holds (signalled as each capture response lands) or [timeoutMs] elapses. private fun awaitBatch( timeoutMs: Long, @@ -247,8 +263,9 @@ fun main() { sdk_name = "posthog-android", sdk_version = PostHogConfig(projectToken = "").sdkVersion, adapter_version = "1.0.0", - // Opt into the capture suites (android posts /i/v1/analytics/events with gzip). - capabilities = listOf("capture_v1", "encoding_gzip", "event_options"), + // Opt into the capture suites (android posts /i/v1/analytics/events and + // /i/v1/ai/events with gzip). + capabilities = listOf("capture_v1", "capture_ai_v1", "encoding_gzip", "event_options"), ), ) } @@ -272,6 +289,7 @@ fun main() { // close() leaves queue files on disk; wipe so they don't replay into this test. File(QUEUE_STORAGE_PREFIX).deleteRecursively() + File("$QUEUE_STORAGE_PREFIX-ai").deleteRecursively() // Create new config val flushIntervalMs = req.flush_interval_ms ?: 100 @@ -301,11 +319,14 @@ fun main() { req.max_retries?.let { config.pushNotificationSubscriptionMaxRetries = it } req.enable_compression?.let { - config.compression = if (it) PostHogCompression.GZIP else PostHogCompression.NONE + val compression = if (it) PostHogCompression.GZIP else PostHogCompression.NONE + config.compression = compression + config.captureAiCompression = compression } - // Set storage prefix for file-backed queue + // Set storage prefixes for the file-backed queues (both wiped on /init and /reset) config.storagePrefix = QUEUE_STORAGE_PREFIX + config.aiStoragePrefix = "$QUEUE_STORAGE_PREFIX-ai" // Set minimal context to provide $lib and $lib_version config.context = TestPostHogContext("posthog-android", config.sdkVersion) @@ -350,6 +371,7 @@ fun main() { event = req.event, distinctId = req.distinct_id, properties = properties, + timestamp = parseHarnessTimestamp(req.timestamp), options = req.options, ) @@ -371,6 +393,38 @@ fun main() { call.respond(CaptureResponse(success = true, uuid = uuid)) } + // Same request as /capture, sent with captureAi to /i/v1/ai/events. Ktor's Netty engine + // streams request bodies without a size limit, so an 8 MiB AI event is received whole. + post("/capture_ai") { + val req = call.receive() + + val ph = + AdapterContext.lock.withLock { + AdapterContext.postHog + } + + if (ph == null) { + call.respond(HttpStatusCode.BadRequest, mapOf("error" to "SDK not initialized")) + return@post + } + + val uuid = + ph.captureAi( + event = req.event, + distinctId = req.distinct_id, + properties = req.properties, + timestamp = parseHarnessTimestamp(req.timestamp), + options = req.options, + ) + + AdapterContext.lock.withLock { + AdapterContext.state.totalEventsCaptured++ + AdapterContext.state.pendingEvents++ + } + + call.respond(CaptureResponse(success = true, uuid = (uuid ?: UUID.randomUUID()).toString())) + } + post("/get_feature_flag") { val req = call.receive() @@ -474,6 +528,7 @@ fun main() { // close() leaves queue files on disk; wipe so they don't replay next test (matches iOS). File(QUEUE_STORAGE_PREFIX).deleteRecursively() + File("$QUEUE_STORAGE_PREFIX-ai").deleteRecursively() AdapterContext.state.pendingEvents = 0 AdapterContext.state.totalEventsCaptured = 0 From 423bb809cbea667b40686de17fdfb6ce8d1e707a Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Sat, 24 Oct 2026 13:57:09 -0400 Subject: [PATCH 04/19] docs: changeset bullets and migration guide for captureAi Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DRbQPidntd4hVufNpMT19L --- .changeset/android-v4-major.md | 3 +++ docs/migration-3.x-to-4.0.md | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/.changeset/android-v4-major.md b/.changeset/android-v4-major.md index 946bd027d..15d127571 100644 --- a/.changeset/android-v4-major.md +++ b/.changeset/android-v4-major.md @@ -25,7 +25,10 @@ - **Breaking:** registered super properties now override the properties the SDK sets itself, such as `$os_name`; super and event properties also override `$is_identified`, `$process_person_profile` and the session replay debug properties, which the SDK no longer re-applies after the caller's values - **Breaking:** the SDK sets the `process_person_profile` option to `false` only when person processing is off, and no longer adds a `$process_person_profile` property to every event - **Breaking:** keep the uuid of its own capture on the event `beforeSend` returns, so a uuid set in `beforeSend`, for example with `copy(uuid = …)`, is replaced +- **Breaking:** add `captureAi` to send AI events such as `$ai_generation` to PostHog's AI endpoint (`/i/v1/ai/events`), which accepts events up to 8 MiB, on its own disk queue; it returns the event uuid, and classes that implement `PostHogInterface` add the method - use the current distinct ID when the `distinctId` argument to `capture` is empty or whitespace-only, instead of dropping the event - **Breaking:** skip deep link query parameters whose name starts with `$` in the `Deep Link Opened` event, so a link can't set a PostHog property such as `$process_person_profile` - stop storing or matching the person-properties deduplication hash for a `$set` sent with `process_person_profile: false` - debug logs for capture requests name the event count and the response status instead of printing the request and response bodies +- add `captureAiCompression`, `captureAiMaxQueueSize`, `captureAiTimeout` and `captureAiMaxEventBytes` to `PostHogConfig` to tune AI requests separately from other events; the defaults are gzip, 100 queued events (and at most 50 MiB), 60 seconds and 8 MiB +- cap a request at about 5 MiB of events, for analytics and AI events, and send as soon as that much is queued; an event larger than that is sent on its own diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index 4550a7ee7..f632641e4 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -151,6 +151,24 @@ When the event is sent, the SDK removes each one from `properties` and uses it f - The event `beforeSend` returns keeps the uuid of the capture it came from. A uuid you set in `beforeSend`, for example with `copy(uuid = …)`, is replaced, so a hook that returns an event kept from an earlier call still sends one uuid per capture. - Events still queued on disk by 3.x are sent to capture v1 after the update, with their legacy properties moved to options. +## AI events + +4.0 adds `captureAi`, which sends AI events, such as `$ai_generation`, to PostHog's AI endpoint (`/i/v1/ai/events`) instead of the analytics endpoint. +It takes the same arguments as `capture`, and the events go through the same `beforeSend` hooks. + +```kotlin +PostHog.captureAi("$ai_generation", properties = mapOf("$ai_model" to "gpt-5", "$ai_trace_id" to traceId)) +``` + +- `captureAi` returns the event's uuid, or `null` when the event isn't queued: the SDK is disabled or opted out, or `beforeSend` dropped it. +- An AI event can be up to 8 MiB. The SDK drops a larger event when it writes it to the queue and logs the event name and size, never its content. Set `captureAiMaxEventBytes` to lower the limit. +- PostHog drops an event sent with `captureAi` whose name doesn't start with `$ai_`. `capture` never sends events to the AI endpoint, even ones named `$ai_*`. +- AI requests time out after 60 seconds, or `captureAiTimeout`. Other requests keep OkHttp's 10-second defaults. +- AI events are queued on disk separately from other events (`posthog-disk-ai-queue` in the app's cache directory). Up to 100 events (`captureAiMaxQueueSize`) and 50 MiB are kept, and the oldest are dropped first. `maxQueueSize` doesn't apply. +- AI requests are gzipped unless you set `captureAiCompression` to `PostHogCompression.NONE`. `compression` doesn't apply. +- AI events use `flushAt`, `maxBatchSize` and `flushIntervalSeconds`, like other events. A request carries up to about 5 MiB, as for analytics events (see [Retries](#retries)). +- The SDK sends AI events unchanged. It doesn't redact or truncate prompts, outputs or media, so use `beforeSend` to remove anything you don't want to send. + ## Retries - The SDK retries network errors and 408, 500, 502, 503 and 504 responses. Other statuses drop the events, including 429. 3.x also retried 429 and 3xx responses. Capture itself doesn't return 429, but a proxy in front of it can. @@ -160,4 +178,5 @@ When the event is sent, the SDK removes each one from `properties` and uses it f - After a failed send, the wait before the next one doubles from 1 second up to 30 seconds, with up to 25% random jitter, rounded to whole seconds. In 3.x it doubled without jitter and a `Retry-After` replaced it. This applies to events, session replay and logs. - Event, session replay and log retries have no limit, as in 3.x: queued records stay until they're delivered or a full queue evicts them (`maxQueueSize`, or `logs.maxBufferSize` for logs). - `maxRetries` is renamed to `pushNotificationSubscriptionMaxRetries`, with no deprecated alias. It only limits push-notification subscription retries. +- A request carries at most `maxBatchSize` events and about 5 MiB, and the SDK sends as soon as about 5 MiB of events is queued, even before `flushAt` is reached. In 3.x, batches were limited by event count only. - A 413 still splits the batch into smaller requests. From 0fa7a6f46a978854019a8dbc6e75f5fe774ea90f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Mon, 26 Oct 2026 14:38:32 -0400 Subject: [PATCH 05/19] docs: list the AI endpoint in the migration guide's endpoint table Co-Authored-By: Claude Opus 5.5 --- docs/migration-3.x-to-4.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index f632641e4..90a926796 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -33,6 +33,7 @@ You need to change something if your app does any of these: | --- | --- | --- | | Analytics events | `/batch` | `/i/v1/analytics/events` | | Session replay | `/s/` | `/s/`, unchanged | +| AI events (`captureAi`) | – | `/i/v1/ai/events` (see [AI events](#ai-events)) | | Logs | `/i/v1/logs` | `/i/v1/logs`, unchanged | | Feature flags | `/flags` | `/flags`, unchanged | From 0a5fa6c8a57931d7fda98d4cdae105579cc6f40a Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 11:52:52 -0400 Subject: [PATCH 06/19] fix: check the AI event size before queuing and return null when over The queue measured an AI event after `encryption`, so an encryption that grows the data dropped events PostHog would accept, and captureAi had already returned the uuid. captureAi now measures the serialized event before encryption and returns null when it's over the limit. The queue's per-record limit had no other user and is removed. Co-Authored-By: Claude Opus 5.5 --- posthog/src/main/java/com/posthog/PostHog.kt | 19 +++++++++++ .../java/com/posthog/internal/EndpointSpec.kt | 31 ++++++++++++++--- .../java/com/posthog/internal/PostHogQueue.kt | 13 +------ .../java/com/posthog/PostHogCaptureAiTest.kt | 34 ++++++++++++++++--- .../internal/PostHogQueueByteLimitsTest.kt | 27 +++------------ 5 files changed, 80 insertions(+), 44 deletions(-) diff --git a/posthog/src/main/java/com/posthog/PostHog.kt b/posthog/src/main/java/com/posthog/PostHog.kt index 0cb5db421..1ab3d2320 100644 --- a/posthog/src/main/java/com/posthog/PostHog.kt +++ b/posthog/src/main/java/com/posthog/PostHog.kt @@ -33,10 +33,12 @@ import com.posthog.internal.PostHogSendCachedEventsIntegration import com.posthog.internal.PostHogSerializer import com.posthog.internal.PostHogSessionManager import com.posthog.internal.PostHogThreadFactory +import com.posthog.internal.captureAiMaxStoredEventBytes import com.posthog.internal.errortracking.PostHogExceptionStepsBuffer import com.posthog.internal.logIntegrationFailure import com.posthog.internal.personPropertiesContext import com.posthog.internal.replay.PostHogSessionReplayHandler +import com.posthog.internal.serializedSize import com.posthog.internal.sortMapRecursively import com.posthog.internal.surveys.PostHogSurveysHandler import com.posthog.logs.PostHogLogRecord @@ -1055,6 +1057,21 @@ public class PostHog private constructor( ) } + /** + * Whether [event], serialized and before [PostHogConfig.encryption], is over the AI event limit. + * Logs the name and size only: AI properties can hold prompts and media. + */ + private fun isOverAiEventLimit(event: PostHogEvent): Boolean { + val config = config ?: return false + val size = config.serializedSize(event) + val limit = config.captureAiMaxStoredEventBytes + if (size <= limit) { + return false + } + config.logger.log("Dropping AI event ${event.event} of $size bytes, over the $limit byte limit.") + return true + } + /** * Enriches and queues one event. Returns its uuid once it's handed to the queue, or null when * it isn't. [aiLane] sends it through the AI queue instead of the analytics (or replay) queue. @@ -1186,6 +1203,8 @@ public class PostHog private constructor( // beforeSend may have opted out if (isOptedOut()) { config?.logger?.log("PostHog is in OptOut state.") + } else if (aiLane && isOverAiEventLimit(postHogEvent)) { + // dropped here rather than in the queue, so captureAi returns null } else if (targetQueue != null) { targetQueue.add(postHogEvent) queued = true diff --git a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt index f9cbda7b0..fbc0e816d 100644 --- a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt +++ b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt @@ -23,8 +23,7 @@ import java.io.OutputStream * request, a record larger than the limit goes alone, and the queue flushes as * soon as that many bytes are queued. [maxQueueBytes] is a FIFO limit on the * queued files on top of [maxQueueSize]; the newest record is always kept. - * [maxRecordBytes] drops a larger record when it is queued. Null means no - * limit. + * Null means no limit. * * The queue itself stays record-type-agnostic; everything that varies * between endpoints lives here. @@ -47,7 +46,6 @@ public class EndpointSpec internal constructor( internal val recordId: ((Record) -> String?)? = null, internal val maxBatchBytes: Long? = null, internal val maxQueueBytes: Long? = null, - internal val maxRecordBytes: Long? = null, ) { public companion object { @JvmStatic @@ -109,7 +107,6 @@ public class EndpointSpec internal constructor( recordId = { event -> event.uuid?.toString() }, maxBatchBytes = CAPTURE_V1_MAX_BATCH_BYTES, maxQueueBytes = AI_MAX_QUEUE_BYTES, - maxRecordBytes = config.captureAiMaxStoredEventBytes, ) @JvmStatic @@ -223,7 +220,31 @@ internal val PostHogConfig.effectiveCaptureAiTimeoutSeconds: Int get() = if (captureAiTimeout > 0) captureAiTimeout else PostHogConfig.DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS /** - * Stored-size limit for an AI event: [PostHogConfig.captureAiMaxEventBytes], capped at PostHog's + * Bytes [event] takes serialized as the queue stores it, before any [PostHogConfig.encryption]. + * Counted as it's written, so the event isn't held in memory a second time. + */ +internal fun PostHogConfig.serializedSize(event: PostHogEvent): Long { + var count = 0L + val counter = + object : OutputStream() { + override fun write(b: Int) { + count++ + } + + override fun write( + b: ByteArray, + off: Int, + len: Int, + ) { + count += len + } + } + serializer.serialize(event, counter.writer().buffered()) + return count +} + +/** + * Size limit for a serialized AI event, before any [PostHogConfig.encryption]: [PostHogConfig.captureAiMaxEventBytes], capped at PostHog's * 8 MiB limit, plus the envelope allowance. */ internal val PostHogConfig.captureAiMaxStoredEventBytes: Long diff --git a/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt b/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt index 8608c7330..455b65b09 100644 --- a/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt +++ b/posthog/src/main/java/com/posthog/internal/PostHogQueue.kt @@ -103,19 +103,8 @@ public class PostHogQueue( os.use { theOutputStream -> spec.encode(record, theOutputStream) } - val size = file.length() - val maxRecordBytes = spec.maxRecordBytes - if (maxRecordBytes != null && size > maxRecordBytes) { - // Name and size only: the record can hold prompts and media. - config.logger.log("Dropping ${spec.describe(record)} of $size bytes, over the $maxRecordBytes byte limit.") - // Not counted in queuedBytes yet, so only the deque entry goes. - synchronized(dequeLock) { - deque.remove(file) - } - file.deleteSafely(config) - return false - } if (trackBytes) { + val size = file.length() synchronized(dequeLock) { queuedBytes += size } diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index 476fdb62d..076404b35 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -8,6 +8,9 @@ import okhttp3.mockwebserver.RecordedRequest import org.junit.Rule import org.junit.rules.TemporaryFolder import java.io.File +import java.io.InputStream +import java.io.OutputStream +import java.util.Base64 import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import kotlin.test.AfterTest @@ -179,16 +182,15 @@ internal class PostHogCaptureAiTest { } @Test - fun `an AI event over captureAiMaxEventBytes is dropped when queued and logs only its name and size`() { + fun `an AI event over captureAiMaxEventBytes returns null and logs only its name and size`() { val http = httpServers.mockHttp() - // the stored limit is the configured bytes plus a 64 KiB envelope + // the limit is the configured bytes plus a 64 KiB envelope val sut = getSut(http) { captureAiMaxEventBytes = 1 } val prompt = "p".repeat(70_000) val uuid = sut.captureAi("\$ai_generation", properties = mapOf("\$ai_input" to prompt)) - // handed to the queue, which drops it on its own thread - assertNotNull(uuid) + assertNull(uuid) aiExecutor.awaitExecution() sut.flush() aiExecutor.awaitExecution() @@ -202,6 +204,30 @@ internal class PostHogCaptureAiTest { assertTrue(logger.messages.none { it.contains("pppp") }) } + @Test + fun `the AI event limit is measured before encryption`() { + val http = httpServers.mockHttp() + val sut = + getSut(http) { + captureAiMaxEventBytes = 1 + // Base64 stores a third more bytes than it's given + encryption = + object : PostHogEncryption { + override fun encrypt(outputStream: OutputStream): OutputStream = Base64.getEncoder().wrap(outputStream) + + override fun decrypt(inputStream: InputStream): InputStream = Base64.getDecoder().wrap(inputStream) + } + } + // under the 64 KiB + 1 byte limit as JSON, over it once encrypted + val prompt = "p".repeat(60_000) + + val uuid = sut.captureAi("\$ai_generation", properties = mapOf("\$ai_input" to prompt)) + + assertNotNull(uuid) + aiExecutor.awaitExecution() + assertEquals(uuid.toString(), events(takeRequest(http)).single()["uuid"]) + } + @Test fun `a name that doesn't start with ai is logged and sent unchanged`() { val http = httpServers.mockHttp() diff --git a/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt index 34c5dc58e..d89344426 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt @@ -46,7 +46,6 @@ internal class PostHogQueueByteLimitsTest { private fun queue( maxBatchBytes: Long? = null, maxQueueBytes: Long? = null, - maxRecordBytes: Long? = null, flushAt: Int = 50, maxQueueSize: Int = 1000, ): PostHogQueue { @@ -77,7 +76,6 @@ internal class PostHogQueueByteLimitsTest { isRetriableStatusCode = { false }, maxBatchBytes = maxBatchBytes, maxQueueBytes = maxQueueBytes, - maxRecordBytes = maxRecordBytes, ) return PostHogQueue(config, spec, executor) } @@ -134,21 +132,6 @@ internal class PostHogQueueByteLimitsTest { assertEquals(listOf("b"), queuedIds(sut)) } - @Test - fun `drops a record over maxRecordBytes, logging its description and size`() { - val sut = queue(maxRecordBytes = 50, maxQueueBytes = 1000, flushAt = 50) - - sut.add(record("big", 80)) - sut.add(record("small", 20)) - executor.awaitExecution() - - assertEquals(listOf("small"), queuedIds(sut)) - assertEquals(1, File(storagePrefix, PROJECT_TOKEN).listFiles()!!.size) - // the dropped record was never counted, so it must not be subtracted either - assertEquals(sut.dequeList.single().length(), sut.queuedBytesForTesting) - assertEquals(listOf("Dropping blob big of 80 bytes, over the 50 byte limit."), logger.messages.filter { it.startsWith("Dropping") }) - } - @Test fun `evicts the oldest records over maxQueueBytes when a record is added`() { val sut = queue(maxQueueBytes = 100, flushAt = 50) @@ -188,19 +171,17 @@ internal class PostHogQueueByteLimitsTest { } @Test - fun `the analytics spec caps a request at 5 MiB and the AI spec adds the disk and event caps`() { + fun `the analytics spec caps a request at 5 MiB and the AI spec adds the disk cap`() { val config = PostHogConfig(PROJECT_TOKEN) val api = PostHogApi(config) val batch = EndpointSpec.batch(config, api, null) assertEquals(5L * 1024 * 1024, batch.maxBatchBytes) assertEquals(null, batch.maxQueueBytes) - assertEquals(null, batch.maxRecordBytes) val ai = EndpointSpec.ai(config, api, null) assertEquals(5L * 1024 * 1024, ai.maxBatchBytes) assertEquals(50L * 1024 * 1024, ai.maxQueueBytes) - assertEquals(8L * 1024 * 1024 + 64 * 1024, ai.maxRecordBytes) assertEquals(100, ai.maxQueueSize(config)) } @@ -210,11 +191,11 @@ internal class PostHogQueueByteLimitsTest { val api = PostHogApi(config) config.captureAiMaxEventBytes = 1024 - assertEquals(1024L + 64 * 1024, EndpointSpec.ai(config, api, null).maxRecordBytes) + assertEquals(1024L + 64 * 1024, config.captureAiMaxStoredEventBytes) config.captureAiMaxEventBytes = 16 * 1024 * 1024 - assertEquals(8L * 1024 * 1024 + 64 * 1024, EndpointSpec.ai(config, api, null).maxRecordBytes) + assertEquals(8L * 1024 * 1024 + 64 * 1024, config.captureAiMaxStoredEventBytes) config.captureAiMaxEventBytes = 0 - assertEquals(8L * 1024 * 1024 + 64 * 1024, EndpointSpec.ai(config, api, null).maxRecordBytes) + assertEquals(8L * 1024 * 1024 + 64 * 1024, config.captureAiMaxStoredEventBytes) config.captureAiMaxQueueSize = 0 assertEquals(100, EndpointSpec.ai(config, api, null).maxQueueSize(config)) From 10842594169a665c11ab5ff94d29e558137286b3 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 11:52:52 -0400 Subject: [PATCH 07/19] docs: describe the AI size limit and timeout as they behave The limit covers an AI event's properties up to 8 MiB, with 64 KiB for the rest of the serialized event, measured before encryption. captureAiTimeout is a timeout without network activity, not a call timeout. Co-Authored-By: Claude Opus 5.5 --- .changeset/android-v4-major.md | 2 +- docs/migration-3.x-to-4.0.md | 10 +++++----- posthog/src/main/java/com/posthog/PostHogConfig.kt | 9 +++++---- posthog/src/main/java/com/posthog/PostHogInterface.kt | 6 +++--- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.changeset/android-v4-major.md b/.changeset/android-v4-major.md index 15d127571..78758d058 100644 --- a/.changeset/android-v4-major.md +++ b/.changeset/android-v4-major.md @@ -25,7 +25,7 @@ - **Breaking:** registered super properties now override the properties the SDK sets itself, such as `$os_name`; super and event properties also override `$is_identified`, `$process_person_profile` and the session replay debug properties, which the SDK no longer re-applies after the caller's values - **Breaking:** the SDK sets the `process_person_profile` option to `false` only when person processing is off, and no longer adds a `$process_person_profile` property to every event - **Breaking:** keep the uuid of its own capture on the event `beforeSend` returns, so a uuid set in `beforeSend`, for example with `copy(uuid = …)`, is replaced -- **Breaking:** add `captureAi` to send AI events such as `$ai_generation` to PostHog's AI endpoint (`/i/v1/ai/events`), which accepts events up to 8 MiB, on its own disk queue; it returns the event uuid, and classes that implement `PostHogInterface` add the method +- **Breaking:** add `captureAi` to send AI events such as `$ai_generation` to PostHog's AI endpoint (`/i/v1/ai/events`), which accepts events whose properties are up to 8 MiB, on its own disk queue; it returns the event uuid, or `null` when the event isn't queued, and classes that implement `PostHogInterface` add the method - use the current distinct ID when the `distinctId` argument to `capture` is empty or whitespace-only, instead of dropping the event - **Breaking:** skip deep link query parameters whose name starts with `$` in the `Deep Link Opened` event, so a link can't set a PostHog property such as `$process_person_profile` - stop storing or matching the person-properties deduplication hash for a `$set` sent with `process_person_profile: false` diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index 90a926796..ffb9ac6fb 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -51,9 +51,9 @@ PostHog Cloud (`us.i.posthog.com`, `eu.i.posthog.com`) serves it. If you send events through your own reverse proxy, update it before you upgrade: -- Forward `/i/v1/analytics/events` to PostHog. +- Forward `/i/v1/analytics/events` and `/i/v1/ai/events` to PostHog. - Pass `Authorization` (the project token, sent as `Bearer `), every `PostHog-*` header, `Content-Type`, `Content-Encoding` and `User-Agent` through unchanged. PostHog answers `401` without `Authorization` and `400` without the others, and the events are dropped. -- Accept request bodies up to 20 MB. nginx and ingress-nginx default to 1 MB. A `413` from the proxy makes the SDK halve the batch until it fits, and drops an event that doesn't fit on its own. +- Accept request bodies up to 20 MB. AI requests are often larger than 1 MB, the default limit in nginx and ingress-nginx. A `413` from the proxy makes the SDK halve the batch until it fits, and drops an event that doesn't fit on its own. PostHog's managed reverse proxy already does this. See the [proxy guides](https://posthog.com/docs/advanced/proxy). @@ -161,10 +161,10 @@ It takes the same arguments as `capture`, and the events go through the same `be PostHog.captureAi("$ai_generation", properties = mapOf("$ai_model" to "gpt-5", "$ai_trace_id" to traceId)) ``` -- `captureAi` returns the event's uuid, or `null` when the event isn't queued: the SDK is disabled or opted out, or `beforeSend` dropped it. -- An AI event can be up to 8 MiB. The SDK drops a larger event when it writes it to the queue and logs the event name and size, never its content. Set `captureAiMaxEventBytes` to lower the limit. +- `captureAi` returns the event's uuid, or `null` when the event isn't queued: the SDK is disabled or opted out, `beforeSend` dropped it, or it's over the size limit. +- An AI event's properties can be up to 8 MiB. The SDK drops a serialized event over 8 MiB + 64 KiB, measured before `encryption`, and logs the event name and size, never its content. Set `captureAiMaxEventBytes` to lower the properties limit; the SDK adds 64 KiB to it for the rest of the event. - PostHog drops an event sent with `captureAi` whose name doesn't start with `$ai_`. `capture` never sends events to the AI endpoint, even ones named `$ai_*`. -- AI requests time out after 60 seconds, or `captureAiTimeout`. Other requests keep OkHttp's 10-second defaults. +- AI requests time out after 60 seconds without network activity, or `captureAiTimeout`. Other requests keep OkHttp's 10-second defaults. - AI events are queued on disk separately from other events (`posthog-disk-ai-queue` in the app's cache directory). Up to 100 events (`captureAiMaxQueueSize`) and 50 MiB are kept, and the oldest are dropped first. `maxQueueSize` doesn't apply. - AI requests are gzipped unless you set `captureAiCompression` to `PostHogCompression.NONE`. `compression` doesn't apply. - AI events use `flushAt`, `maxBatchSize` and `flushIntervalSeconds`, like other events. A request carries up to about 5 MiB, as for analytics events (see [Retries](#retries)). diff --git a/posthog/src/main/java/com/posthog/PostHogConfig.kt b/posthog/src/main/java/com/posthog/PostHogConfig.kt index 9550b5c89..188f56283 100644 --- a/posthog/src/main/java/com/posthog/PostHogConfig.kt +++ b/posthog/src/main/java/com/posthog/PostHogConfig.kt @@ -443,8 +443,8 @@ public open class PostHogConfig( public var captureAiMaxQueueSize: Int = DEFAULT_CAPTURE_AI_MAX_QUEUE_SIZE /** - * Timeout in seconds of a [PostHog.captureAi] request: its HTTP client's call, read and write - * timeouts. + * How long, in seconds, a [PostHog.captureAi] request can go without network activity before it + * times out: its HTTP client's read and write timeouts. * * AI requests can carry several MB, so this is longer than for other requests, which keep * OkHttp's 10 second defaults. A value of 0 or less uses the default. @@ -454,8 +454,9 @@ public open class PostHogConfig( public var captureAiTimeout: Int = DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS /** - * Maximum size, in bytes, of a [PostHog.captureAi] event. The SDK drops a larger event when it - * queues it, logging only the event name and size. + * Maximum size, in bytes, of a [PostHog.captureAi] event's properties. The SDK drops a serialized + * event over this plus 64 KiB for the rest of the event, measured before [encryption]; then + * `captureAi` returns null and logs only the event name and size. * * Use it to lower the limit. PostHog drops an AI event whose properties are over 8 MiB, so a * larger value, or a value of 0 or less, uses 8 MiB. diff --git a/posthog/src/main/java/com/posthog/PostHogInterface.kt b/posthog/src/main/java/com/posthog/PostHogInterface.kt index 58e57dcef..9cc064879 100644 --- a/posthog/src/main/java/com/posthog/PostHogInterface.kt +++ b/posthog/src/main/java/com/posthog/PostHogInterface.kt @@ -118,8 +118,8 @@ public interface PostHogInterface : PostHogCoreInterface { * [PostHogConfig.flushIntervalSeconds], like other events, and * [PostHogConfig.captureAiCompression], [PostHogConfig.captureAiMaxQueueSize] and * [PostHogConfig.captureAiTimeout] instead of `compression`, `maxQueueSize` and the default - * timeouts. An event over [PostHogConfig.captureAiMaxEventBytes] (8 MiB by default) is dropped - * when it's queued, with a log line naming the event and its size. Requests carry up to about + * timeouts. A serialized event over [PostHogConfig.captureAiMaxEventBytes] plus 64 KiB (8 MiB + + * 64 KiB by default) is dropped, with a log line naming the event and its size. Requests carry up to about * 5 MiB of events, and the SDK sends as soon as that much is queued. Up to 50 MiB of AI events * are kept on disk; beyond that, the oldest are dropped. * @@ -136,7 +136,7 @@ public interface PostHogInterface : PostHogCoreInterface { * @param timestamp the event timestamp override. UTC is preferred; the equivalent instant is serialized in UTC. If omitted, the current time is used * @param options settings that tell PostHog how to process this event, see `options` in [capture] * @return the event's uuid once it's handed to the queue, or null when it isn't: the SDK is - * disabled or opted out, or `beforeSend` dropped it + * disabled or opted out, `beforeSend` dropped it, or it's over the size limit */ public fun captureAi( event: String, From f153814cb772709019d0a0c5eb7379f7d392ba36 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 11:55:17 -0400 Subject: [PATCH 08/19] chore(compliance): report no uuid when captureAi doesn't queue the event Co-Authored-By: Claude Opus 5.5 --- .../com/posthog/compliance/ComplianceAdapter.kt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt b/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt index 4e151ef6a..1bd55d60e 100644 --- a/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt +++ b/sdk_compliance_adapter/src/main/kotlin/com/posthog/compliance/ComplianceAdapter.kt @@ -93,7 +93,8 @@ data class CaptureRequest( data class CaptureResponse( val success: Boolean, - val uuid: String, + // Null (omitted from the JSON) when the SDK didn't queue the event. + val uuid: String? = null, ) data class FlagRequest( @@ -318,6 +319,8 @@ fun main() { config.httpClient = httpClient req.max_retries?.let { config.pushNotificationSubscriptionMaxRetries = it } + // The harness sends one enable_compression flag and no AI-lane setting, so it sets + // both lanes. With it unset, the AI lane keeps its documented default (gzip). req.enable_compression?.let { val compression = if (it) PostHogCompression.GZIP else PostHogCompression.NONE config.compression = compression @@ -417,12 +420,16 @@ fun main() { options = req.options, ) - AdapterContext.lock.withLock { - AdapterContext.state.totalEventsCaptured++ - AdapterContext.state.pendingEvents++ + // A null uuid means the SDK didn't queue the event (for example it's over the size + // limit), so no request will carry it and no uuid is reported. + if (uuid != null) { + AdapterContext.lock.withLock { + AdapterContext.state.totalEventsCaptured++ + AdapterContext.state.pendingEvents++ + } } - call.respond(CaptureResponse(success = true, uuid = (uuid ?: UUID.randomUUID()).toString())) + call.respond(CaptureResponse(success = true, uuid = uuid?.toString())) } post("/get_feature_flag") { From 941cd6ce8cd76e3ed33655a95f0cb366aea9fcbc Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 12:00:52 -0400 Subject: [PATCH 09/19] fix: log each event capture v1 drops with the reason it gives Debug logs named only the drop count, so a misrouted_event or ai_event_too_big drop was invisible. Each dropped event is now logged with its name, uuid and the server's details, never its properties. Adds a test that an AI event with exactly 8 MiB of properties is sent. Co-Authored-By: Claude Opus 5.5 --- .changeset/android-v4-major.md | 2 +- docs/migration-3.x-to-4.0.md | 2 +- .../java/com/posthog/internal/PostHogApi.kt | 17 +++++++++----- .../java/com/posthog/PostHogCaptureAiTest.kt | 22 ++++++++++++++++++ .../posthog/internal/PostHogCaptureV1Test.kt | 23 +++++++++++++++++++ 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/.changeset/android-v4-major.md b/.changeset/android-v4-major.md index 78758d058..60ec1b526 100644 --- a/.changeset/android-v4-major.md +++ b/.changeset/android-v4-major.md @@ -29,6 +29,6 @@ - use the current distinct ID when the `distinctId` argument to `capture` is empty or whitespace-only, instead of dropping the event - **Breaking:** skip deep link query parameters whose name starts with `$` in the `Deep Link Opened` event, so a link can't set a PostHog property such as `$process_person_profile` - stop storing or matching the person-properties deduplication hash for a `$set` sent with `process_person_profile: false` -- debug logs for capture requests name the event count and the response status instead of printing the request and response bodies +- debug logs for capture requests name the event count and the response status instead of printing the request and response bodies, and name each event PostHog drops with the reason it gives - add `captureAiCompression`, `captureAiMaxQueueSize`, `captureAiTimeout` and `captureAiMaxEventBytes` to `PostHogConfig` to tune AI requests separately from other events; the defaults are gzip, 100 queued events (and at most 50 MiB), 60 seconds and 8 MiB - cap a request at about 5 MiB of events, for analytics and AI events, and send as soon as that much is queued; an event larger than that is sent on its own diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index ffb9ac6fb..267ef2a93 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -70,7 +70,7 @@ If a proxy redirects capture requests to another host, point `host` at the final Capture v1 checks each event on its own and drops only the invalid ones, so the rest of the batch is delivered. The SDK doesn't check these before sending. PostHog drops the event, and the SDK removes it from the queue. -With `debug` on, the SDK logs how many events were dropped. +With `debug` on, the SDK logs the name and uuid of each dropped event and the reason PostHog gives, such as `misrouted_event`, never its properties. - An event with an empty event name is dropped, for example `capture("")`. - An event with an empty or whitespace-only distinct ID is dropped, for example one `beforeSend` sets to `""`. diff --git a/posthog/src/main/java/com/posthog/internal/PostHogApi.kt b/posthog/src/main/java/com/posthog/internal/PostHogApi.kt index 4f52da0c2..2fde08f2b 100644 --- a/posthog/src/main/java/com/posthog/internal/PostHogApi.kt +++ b/posthog/src/main/java/com/posthog/internal/PostHogApi.kt @@ -222,7 +222,7 @@ public class PostHogApi( config.logger.log("Response ${response.request.url}: ${response.code}") if (response.code != 307 && response.code != 308) { - return captureV1Result(response, path, uuids, identity) + return captureV1Result(response, path, events, identity) } val target = response.header("Location")?.let { location -> request.url.resolve(location) } @@ -280,7 +280,7 @@ public class PostHogApi( private fun captureV1Result( response: Response, path: String, - uuids: List, + events: List, identity: CaptureV1RequestIdentity, ): SendResult { if (!response.isSuccessful) { @@ -293,22 +293,27 @@ public class PostHogApi( val results = readCaptureV1Results(response) if (results == null) { config.logger.log( - "Capture returned ${response.code} without readable per-event results, treating ${uuids.size} events as delivered.", + "Capture returned ${response.code} without readable per-event results, treating ${events.size} events as delivered.", ) } val retryUuids = mutableSetOf() var dropped = 0 // Events missing from the results count as delivered. - for (uuid in uuids) { + for (event in events) { + val uuid = event.uuid?.toString() ?: continue val entry = results?.get(uuid) as? Map<*, *> ?: continue when (entry["result"]) { "retry" -> retryUuids.add(uuid) - "drop" -> dropped++ + "drop" -> { + dropped++ + // The event name, uuid and the server's reason (such as misrouted_event or + // ai_event_too_big), never the properties. + config.logger.log("Capture dropped event ${event.event} ($uuid): ${entry["details"] ?: "no details"}.") + } } } - // Counts only: event uuids and server details can identify users. if (dropped > 0 || retryUuids.isNotEmpty()) { config.logger.log("Capture: $dropped dropped, ${retryUuids.size} to retry.") } diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index 076404b35..77546bc9e 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -204,6 +204,28 @@ internal class PostHogCaptureAiTest { assertTrue(logger.messages.none { it.contains("pppp") }) } + @Test + fun `an AI event whose properties are exactly 8 MiB is sent`() { + val ceiling = 8 * 1024 * 1024 + val http = httpServers.mockHttp() + val sut = + getSut(http) { + captureAiCompression = PostHogCompression.NONE + // pin the properties after the SDK's enrichment; {"$ai_input":"…"} is 16 bytes around the value + addBeforeSend { event -> event.copy(properties = mutableMapOf("\$ai_input" to "a".repeat(ceiling - 16))) } + } + + assertNotNull(sut.captureAi("\$ai_generation")) + + aiExecutor.awaitExecution() + val body = takeRequest(http).body.readUtf8() + // the properties object is the event's last field, closed by the event, batch and body + val properties = body.substring(body.indexOf("\"properties\":") + "\"properties\":".length, body.length - "}]}".length) + assertEquals(ceiling, properties.toByteArray(Charsets.UTF_8).size) + assertEquals("\$ai_generation", captureV1Events(body).single()["event"]) + assertTrue(logger.messages.none { it.startsWith("Dropping AI event") }) + } + @Test fun `the AI event limit is measured before encryption`() { val http = httpServers.mockHttp() diff --git a/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt b/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt index 2916d9eb6..f2b4ea503 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt @@ -332,9 +332,32 @@ internal class PostHogCaptureV1Test { assertEquals(setOf(retry.uuid.toString()), result.retryIds) assertEquals(7, result.retryAfterSeconds) assertTrue(logger.messages.contains("Capture: 1 dropped, 1 to retry.")) + assertTrue(logger.messages.contains("Capture dropped event dropped (${dropped.uuid}): no details.")) assertTrue(logger.messages.none { it.contains(retry.uuid.toString()) }) } + @Test + fun `logs each dropped event with the details capture gives, without its properties`() { + val misrouted = event("button_clicked", properties = mapOf("prompt" to "secret prompt")) + val tooBig = event("\$ai_generation", properties = mapOf("prompt" to "secret prompt")) + val http = + httpServers.mockHttp( + response = + MockResponse().setBody( + "{\"results\":{" + + "\"${misrouted.uuid}\":{\"result\":\"drop\",\"details\":\"misrouted_event\"}," + + "\"${tooBig.uuid}\":{\"result\":\"drop\",\"details\":\"ai_event_too_big\"}}}", + ), + ) + + api(http.url("/").toString()).captureAi(listOf(misrouted, tooBig)) + + assertTrue(logger.messages.contains("Capture dropped event button_clicked (${misrouted.uuid}): misrouted_event.")) + assertTrue(logger.messages.contains("Capture dropped event \$ai_generation (${tooBig.uuid}): ai_event_too_big.")) + assertTrue(logger.messages.contains("Capture: 2 dropped, 0 to retry.")) + assertTrue(logger.messages.none { it.contains("secret prompt") }) + } + @Test fun `a 2xx without readable results counts as delivered`() { val http = httpServers.mockHttp(response = MockResponse().setBody("not json")) From 117bc048b90c2156a7be33a7bdfac023b409a67f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 12:10:52 -0400 Subject: [PATCH 10/19] fix: stop measuring an AI event once it passes the size limit captureAi checks the size on the caller's thread, so a huge prompt or base64 payload no longer costs a full serialization just to be dropped. Co-Authored-By: Claude Opus 5.5 --- posthog/src/main/java/com/posthog/PostHog.kt | 8 +++---- .../java/com/posthog/internal/EndpointSpec.kt | 21 ++++++++++++++++--- .../java/com/posthog/PostHogCaptureAiTest.kt | 2 +- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/posthog/src/main/java/com/posthog/PostHog.kt b/posthog/src/main/java/com/posthog/PostHog.kt index 1ab3d2320..614304cac 100644 --- a/posthog/src/main/java/com/posthog/PostHog.kt +++ b/posthog/src/main/java/com/posthog/PostHog.kt @@ -1059,16 +1059,16 @@ public class PostHog private constructor( /** * Whether [event], serialized and before [PostHogConfig.encryption], is over the AI event limit. - * Logs the name and size only: AI properties can hold prompts and media. + * Counting stops at the limit, so rejecting a large payload on the caller's thread costs no more + * than the limit. Logs the name and size only: AI properties can hold prompts and media. */ private fun isOverAiEventLimit(event: PostHogEvent): Boolean { val config = config ?: return false - val size = config.serializedSize(event) val limit = config.captureAiMaxStoredEventBytes - if (size <= limit) { + if (config.serializedSize(event, stopAfter = limit) <= limit) { return false } - config.logger.log("Dropping AI event ${event.event} of $size bytes, over the $limit byte limit.") + config.logger.log("Dropping AI event ${event.event} of more than $limit bytes, over the limit.") return true } diff --git a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt index fbc0e816d..b3500616c 100644 --- a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt +++ b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt @@ -219,16 +219,26 @@ internal val PostHogConfig.effectiveCaptureAiMaxQueueSize: Int internal val PostHogConfig.effectiveCaptureAiTimeoutSeconds: Int get() = if (captureAiTimeout > 0) captureAiTimeout else PostHogConfig.DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS +/** Thrown by [serializedSize]'s counter to stop serializing once the count passes its bound. */ +private object SerializedSizeExceeded : RuntimeException() { + override fun fillInStackTrace(): Throwable = this +} + /** * Bytes [event] takes serialized as the queue stores it, before any [PostHogConfig.encryption]. - * Counted as it's written, so the event isn't held in memory a second time. + * Counted as it's written, so the event isn't held in memory a second time. Stops at + * [stopAfter] + 1, so an oversized event costs no more than the bound to reject. */ -internal fun PostHogConfig.serializedSize(event: PostHogEvent): Long { +internal fun PostHogConfig.serializedSize( + event: PostHogEvent, + stopAfter: Long = Long.MAX_VALUE, +): Long { var count = 0L val counter = object : OutputStream() { override fun write(b: Int) { count++ + if (count > stopAfter) throw SerializedSizeExceeded } override fun write( @@ -237,9 +247,14 @@ internal fun PostHogConfig.serializedSize(event: PostHogEvent): Long { len: Int, ) { count += len + if (count > stopAfter) throw SerializedSizeExceeded } } - serializer.serialize(event, counter.writer().buffered()) + try { + serializer.serialize(event, counter.writer().buffered()) + } catch (e: SerializedSizeExceeded) { + return stopAfter + 1 + } return count } diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index 77546bc9e..768b2a3ff 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -198,7 +198,7 @@ internal class PostHogCaptureAiTest { assertTrue(aiQueueFiles().isEmpty()) val dropped = logger.messages.single { it.startsWith("Dropping AI event") } assertTrue( - dropped.matches(Regex("Dropping AI event \\\$ai_generation of \\d+ bytes, over the ${1 + 64 * 1024} byte limit\\.")), + dropped.matches(Regex("Dropping AI event \\\$ai_generation of more than ${1 + 64 * 1024} bytes, over the limit\\.")), dropped, ) assertTrue(logger.messages.none { it.contains("pppp") }) From f219836ca971f1f487ea3298313dbf9b059abe72 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 12:10:52 -0400 Subject: [PATCH 11/19] fix: log an unserializable property by key and type, not by value Properties can hold prompts and other user content. Co-Authored-By: Claude Opus 5.5 --- .../posthog/internal/GsonSafeMapSerializer.kt | 5 ++-- .../posthog/internal/PostHogSerializerTest.kt | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt b/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt index 103f6d222..f8ef422b9 100644 --- a/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt +++ b/posthog/src/main/java/com/posthog/internal/GsonSafeMapSerializer.kt @@ -59,9 +59,10 @@ internal class GsonSafeMapSerializer(private val config: PostHogConfig) : JsonSe try { config.serializer.gson.toJsonTree(targetValue) } catch (e: Throwable) { + // the value's type only: properties can hold prompts and other user content config.logger.log( - "Property '$key' with value '$targetValue' cannot be serialized to JSON: $e. " + - "This property will be ignored.", + "Property '$key' of type ${targetValue.javaClass.name} cannot be serialized to JSON " + + "(${e.javaClass.simpleName}). This property will be ignored.", ) null } diff --git a/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt index cd7a5da72..19daaf390 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt @@ -3,6 +3,7 @@ package com.posthog.internal import com.posthog.PROJECT_TOKEN import com.posthog.PostHogConfig import com.posthog.PostHogEvent +import com.posthog.TestLogger import com.posthog.date import com.posthog.generateEvent import org.junit.Rule @@ -313,4 +314,28 @@ internal class PostHogSerializerTest { assertEquals(expectedJson, actualJson) } + + private data class Turn(val prompt: String, val score: Double) + + @Test + fun `an unserializable property is logged by key and type, never by value`() { + val logger = TestLogger() + val config = PostHogConfig(PROJECT_TOKEN).apply { this.logger = logger } + val event = generateEvent().copy(properties = mutableMapOf("turn" to Turn("SECRET-PROMPT", Double.NaN))) + + config.serializer.serialize(event, StringWriter()) + + val message = logger.messages.single { it.contains("cannot be serialized") } + assertTrue(message.contains("'turn'"), message) + assertTrue(logger.messages.none { it.contains("SECRET-PROMPT") }) + } + + @Test + fun `serializedSize stops counting past its bound`() { + val config = PostHogConfig(PROJECT_TOKEN) + val event = generateEvent().copy(properties = mutableMapOf("big" to "a".repeat(1_000_000))) + + assertEquals(1_001L, config.serializedSize(event, stopAfter = 1_000)) + assertTrue(config.serializedSize(event) > 1_000_000) + } } From d44bc39a714987f3822b7709aeac484d448a70d7 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Tue, 27 Oct 2026 14:38:57 -0400 Subject: [PATCH 12/19] docs: say the AI size check builds the full JSON tree before it stops counting --- posthog/src/main/java/com/posthog/PostHog.kt | 5 +++-- posthog/src/main/java/com/posthog/internal/EndpointSpec.kt | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/posthog/src/main/java/com/posthog/PostHog.kt b/posthog/src/main/java/com/posthog/PostHog.kt index 614304cac..97eff37e3 100644 --- a/posthog/src/main/java/com/posthog/PostHog.kt +++ b/posthog/src/main/java/com/posthog/PostHog.kt @@ -1059,8 +1059,9 @@ public class PostHog private constructor( /** * Whether [event], serialized and before [PostHogConfig.encryption], is over the AI event limit. - * Counting stops at the limit, so rejecting a large payload on the caller's thread costs no more - * than the limit. Logs the name and size only: AI properties can hold prompts and media. + * Runs on the caller's thread so captureAi can return null; writing stops at the limit, but the + * properties JSON tree is built in full first. Logs the name and size only: AI properties can + * hold prompts and media. */ private fun isOverAiEventLimit(event: PostHogEvent): Boolean { val config = config ?: return false diff --git a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt index b3500616c..81fe9cf9e 100644 --- a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt +++ b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt @@ -226,8 +226,8 @@ private object SerializedSizeExceeded : RuntimeException() { /** * Bytes [event] takes serialized as the queue stores it, before any [PostHogConfig.encryption]. - * Counted as it's written, so the event isn't held in memory a second time. Stops at - * [stopAfter] + 1, so an oversized event costs no more than the bound to reject. + * Counted as it's written, so the event isn't held in memory a second time. Stops writing at + * [stopAfter] + 1, though the serializer still builds the whole properties JSON tree first. */ internal fun PostHogConfig.serializedSize( event: PostHogEvent, From 2d22dbb8ffced834fd67d548557e9a290fee9ce1 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 28 Oct 2026 10:48:08 -0400 Subject: [PATCH 13/19] test: double bytes instead of Base64 in the AI encryption test java.util.Base64 isn't on API 23, so animalsnifferTest failed. Co-Authored-By: Claude Opus 5.5 --- .../java/com/posthog/PostHogCaptureAiTest.kt | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index 768b2a3ff..b5e4bbd79 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -8,9 +8,10 @@ import okhttp3.mockwebserver.RecordedRequest import org.junit.Rule import org.junit.rules.TemporaryFolder import java.io.File +import java.io.FilterInputStream +import java.io.FilterOutputStream import java.io.InputStream import java.io.OutputStream -import java.util.Base64 import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import kotlin.test.AfterTest @@ -232,12 +233,35 @@ internal class PostHogCaptureAiTest { val sut = getSut(http) { captureAiMaxEventBytes = 1 - // Base64 stores a third more bytes than it's given + // writes every byte twice, so the stored event is double its JSON size encryption = object : PostHogEncryption { - override fun encrypt(outputStream: OutputStream): OutputStream = Base64.getEncoder().wrap(outputStream) - - override fun decrypt(inputStream: InputStream): InputStream = Base64.getDecoder().wrap(inputStream) + override fun encrypt(outputStream: OutputStream): OutputStream = + object : FilterOutputStream(outputStream) { + override fun write(b: Int) { + out.write(b) + out.write(b) + } + } + + override fun decrypt(inputStream: InputStream): InputStream = + object : FilterInputStream(inputStream) { + override fun read(): Int = super.read().also { super.read() } + + override fun read( + b: ByteArray, + off: Int, + len: Int, + ): Int { + var n = 0 + while (n < len) { + val c = read() + if (c < 0) break + b[off + n++] = c.toByte() + } + return if (n == 0 && len > 0) -1 else n + } + } } } // under the 64 KiB + 1 byte limit as JSON, over it once encrypted From 2912dccc77908bf2098349b7dede54be0551a93f Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 28 Oct 2026 10:48:09 -0400 Subject: [PATCH 14/19] test: make the unserializable Turn a plain class The data class hashCode calls Double.hashCode(double), which isn't on API 23. Co-Authored-By: Claude Opus 5.5 --- .../test/java/com/posthog/internal/PostHogSerializerTest.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt index 19daaf390..256704b5d 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt @@ -315,7 +315,8 @@ internal class PostHogSerializerTest { assertEquals(expectedJson, actualJson) } - private data class Turn(val prompt: String, val score: Double) + // not a data class: its generated hashCode calls Double.hashCode(double), missing on API 23 + private class Turn(val prompt: String, val score: Double) @Test fun `an unserializable property is logged by key and type, never by value`() { From b3e29fb4bd08050bc976e1a4140f7f82df9eccec Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 28 Oct 2026 11:47:52 -0400 Subject: [PATCH 15/19] test: write encrypted test streams in blocks Co-Authored-By: Claude Opus 5.5 --- .../test/java/com/posthog/PostHogCaptureAiTest.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index b5e4bbd79..65da97680 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -242,6 +242,19 @@ internal class PostHogCaptureAiTest { out.write(b) out.write(b) } + + override fun write( + b: ByteArray, + off: Int, + len: Int, + ) { + val doubled = ByteArray(len * 2) + for (i in 0 until len) { + doubled[2 * i] = b[off + i] + doubled[2 * i + 1] = b[off + i] + } + out.write(doubled) + } } override fun decrypt(inputStream: InputStream): InputStream = From f4da7c87d997bbdeaa38f0735ebf5a7d478b5b4d Mon Sep 17 00:00:00 2001 From: Anna Garcia <11654201+turnipdabeets@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:14:20 -0400 Subject: [PATCH 16/19] fix: retry capture v1 requests that get 404 or 429 (#897) * fix: retry capture v1 requests that get 404 or 429 A 404 usually means a misconfigured proxy and a 429 now only comes from an overloaded ingress, so keep the events queued and retry them with the usual backoff and Retry-After handling instead of dropping them. Matches PostHog/posthog-go#395. * fix: warn once when capture v1 returns 404 A 404 now keeps events queued, so nothing tells the app that the route is missing. Print one warning per client, even without debug, naming the URL and the path a reverse proxy or self-hosted instance must serve. * fix: leave host credentials and query out of the capture v1 404 warning Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .changeset/android-v4-major.md | 4 +- docs/migration-3.x-to-4.0.md | 6 +-- .../java/com/posthog/internal/EndpointSpec.kt | 7 ++-- .../java/com/posthog/internal/PostHogApi.kt | 18 +++++++++ .../java/com/posthog/PostHogCaptureAiTest.kt | 8 ++-- .../com/posthog/internal/EndpointSpecTest.kt | 4 -- .../posthog/internal/PostHogCaptureV1Test.kt | 37 +++++++++++++++++-- .../com/posthog/internal/PostHogQueueTest.kt | 8 ++-- 8 files changed, 69 insertions(+), 23 deletions(-) diff --git a/.changeset/android-v4-major.md b/.changeset/android-v4-major.md index 60ec1b526..d6b6ab1c1 100644 --- a/.changeset/android-v4-major.md +++ b/.changeset/android-v4-major.md @@ -11,8 +11,8 @@ - **Breaking:** return `null` instead of the raw string for a malformed, empty or whitespace-only feature flag payload from `getFeatureFlagResult(key)?.payload`; payloads must be valid JSON (a JSON `null` payload now also reads as `null` instead of the string `"null"`) - **Breaking:** make SDK-internal helpers in the `com.posthog.internal` and `com.posthog.android.replay.internal` packages non-public - **Breaking:** remove the deprecated `remoteConfig` config option; the SDK now always loads remote config at setup -- **Breaking:** send events to capture v1 (`/i/v1/analytics/events`) with the project token in an `Authorization: Bearer` header; if you use a reverse proxy or self-host PostHog, make sure the path is served, because a 404 or other non-retried response drops the events, and authenticate your proxy with a header other than `Authorization`, which capture requests replace -- **Breaking:** stop retrying 429 and 3xx responses for events, and cap a `Retry-After` pause at 30 seconds for events, session replay and logs +- **Breaking:** send events to capture v1 (`/i/v1/analytics/events`) with the project token in an `Authorization: Bearer` header; if you use a reverse proxy or self-host PostHog, make sure the path is served, because events stay queued and are retried on a 404, with a warning printed once even when `debug` is off, and authenticate your proxy with a header other than `Authorization`, which capture requests replace +- **Breaking:** stop retrying 3xx responses for events, retry 404 responses, and cap a `Retry-After` pause at 30 seconds for events, session replay and logs - **Breaking:** add up to 25% random jitter to the wait after a failed send, which still doubles from 1 second to 30 seconds, and treat a `Retry-After` as a minimum wait instead of replacing the backoff, for events, session replay and logs; queued records are still kept until they're delivered - **Breaking:** follow only a 307 or 308 redirect of a capture request to the same origin as `host`, at most 5 times; any other redirect drops the events, so point `host` at the final host - **Breaking:** send the `$process_person_profile`, `$cookieless_mode`, `$ignore_sent_at` and `$product_tour_id` properties as capture options (`PostHogEvent.options`, a new trailing constructor parameter that Java callers of the full constructor and `copy` must pass), where PostHog drops an event whose option value it can't read diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index 267ef2a93..daf5b3bee 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -24,7 +24,7 @@ You need to change something if your app does any of these: - registers a super property, or passes an event property, with the same key as one the SDK sets, such as `$os_name` or `$is_identified`, and expects the SDK's value to win - calls `capture`, `screen`, `identify`, `setPersonProperties`, `alias`, `group` or `captureException` from Java, implements `PostHogInterface`, or calls `identify` through a `PostHogCoreInterface` reference - sets the event `uuid` in `beforeSend` -- relies on 429 responses being retried, or on a `Retry-After` longer than 30 seconds +- relies on a `Retry-After` longer than 30 seconds - sets `maxRetries` on `PostHogConfig` or `PostHogAndroidConfig` ## Endpoints @@ -57,7 +57,7 @@ If you send events through your own reverse proxy, update it before you upgrade: PostHog's managed reverse proxy already does this. See the [proxy guides](https://posthog.com/docs/advanced/proxy). -The SDK doesn't fall back to `/batch`. A 404, or any other error status that isn't retried, such as 403 or 405, drops the events. +The SDK doesn't fall back to `/batch`. A 404 is retried, so the events stay queued until the path is served. Any other error status that isn't retried, such as 403 or 405, drops the events. ### Redirects @@ -172,7 +172,7 @@ PostHog.captureAi("$ai_generation", properties = mapOf("$ai_model" to "gpt-5", " ## Retries -- The SDK retries network errors and 408, 500, 502, 503 and 504 responses. Other statuses drop the events, including 429. 3.x also retried 429 and 3xx responses. Capture itself doesn't return 429, but a proxy in front of it can. +- The SDK retries network errors and 404, 408, 429, 500, 502, 503 and 504 responses. Other statuses drop the events, including 3xx. 3.x also retried 3xx responses, and didn't retry 404. - When the response asks to retry only some events, the SDK keeps those events queued and removes the rest. - A failed request is resent with the same events, request ID and `created_at`, and the next attempt number, before newer events are sent. OkHttp doesn't retry capture requests on its own. - A `Retry-After` header pauses sending for at most 30 seconds. This also applies to session replay and logs. In 3.x there was no cap. diff --git a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt index 81fe9cf9e..41abd466f 100644 --- a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt +++ b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt @@ -194,10 +194,11 @@ internal class SendResult( ) /** - * Capture v1 retry policy: 408 and the listed 5xx. Everything else, including 429 and 3xx, is - * terminal; 413 is handled by the queue's batch halving. + * Capture v1 retry policy: 404, 408, 429 and the listed 5xx. A 404 usually means a misconfigured + * proxy and a 429 an overloaded ingress, so neither drops events. Everything else, including 3xx, + * is terminal; 413 is handled by the queue's batch halving. */ -internal fun isCaptureV1RetriableStatusCode(code: Int): Boolean = code == 408 || code in setOf(500, 502, 503, 504) +internal fun isCaptureV1RetriableStatusCode(code: Int): Boolean = code in setOf(404, 408, 429, 500, 502, 503, 504) /** Soft request size for the capture v1 analytics and AI endpoints, well under their body limits. */ internal const val CAPTURE_V1_MAX_BATCH_BYTES: Long = 5L * 1024 * 1024 diff --git a/posthog/src/main/java/com/posthog/internal/PostHogApi.kt b/posthog/src/main/java/com/posthog/internal/PostHogApi.kt index 2fde08f2b..318fdee63 100644 --- a/posthog/src/main/java/com/posthog/internal/PostHogApi.kt +++ b/posthog/src/main/java/com/posthog/internal/PostHogApi.kt @@ -31,6 +31,7 @@ import java.net.SocketTimeoutException import java.net.UnknownHostException import java.util.UUID import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean import javax.net.ssl.SSLException /** @@ -62,6 +63,9 @@ public class PostHogApi( } } + // A 404 keeps the events queued, so the warning is printed once instead of on every retry. + private val warnedCaptureNotFound = AtomicBoolean(false) + private val captureMediaType by lazy { try { APP_JSON.toMediaType() @@ -221,6 +225,20 @@ public class PostHogApi( // The status only: the results name event uuids. config.logger.log("Response ${response.request.url}: ${response.code}") + if (response.code == 404 && warnedCaptureNotFound.compareAndSet(false, true)) { + // Warnings log even with debug off, so drop any credentials or query the host carries. + val safeUrl = + response.request.url.newBuilder() + .username("") + .password("") + .query(null) + .fragment(null) + .build() + config.logger.logWarning( + "$safeUrl returned 404. Events stay queued and are retried. " + + "If you use a reverse proxy or self-host PostHog, make sure it serves $path.", + ) + } if (response.code != 307 && response.code != 308) { return captureV1Result(response, path, events, identity) } diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index 65da97680..13b89599b 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -168,18 +168,16 @@ internal class PostHogCaptureAiTest { } @Test - fun `a 404 from the AI endpoint is terminal`() { - val http = httpServers.mockHttp(total = 2, response = MockResponse().setResponseCode(404)) + fun `a 404 from the AI endpoint keeps the event queued`() { + val http = httpServers.mockHttp(response = MockResponse().setResponseCode(404)) val sut = getSut(http) sut.captureAi("\$ai_generation") aiExecutor.awaitExecution() - sut.flush() - aiExecutor.awaitExecution() assertEquals(1, http.requestCount) assertEquals("/i/v1/ai/events", takeRequest(http).path) - assertTrue(aiQueueFiles().isEmpty()) + assertEquals(1, aiQueueFiles().size) } @Test diff --git a/posthog/src/test/java/com/posthog/internal/EndpointSpecTest.kt b/posthog/src/test/java/com/posthog/internal/EndpointSpecTest.kt index 73a68f956..12946becc 100644 --- a/posthog/src/test/java/com/posthog/internal/EndpointSpecTest.kt +++ b/posthog/src/test/java/com/posthog/internal/EndpointSpecTest.kt @@ -274,13 +274,9 @@ internal class EndpointSpecTest { @Test fun `events spec does not retry non-retriable status codes`() { - // Events keeps a curated retry set (408, 500, 502, 503, 504); it must not - // retry 429, 3xx, arbitrary 4xx, nor the extra 5xx codes the broad logs - // policy (500..599) covers. Guards against widening the set by mistake. assertEquals(false, isCaptureV1RetriableStatusCode(307)) assertEquals(false, isCaptureV1RetriableStatusCode(400)) assertEquals(false, isCaptureV1RetriableStatusCode(413)) - assertEquals(false, isCaptureV1RetriableStatusCode(429)) assertEquals(false, isCaptureV1RetriableStatusCode(501)) assertEquals(false, isCaptureV1RetriableStatusCode(505)) assertEquals(false, isCaptureV1RetriableStatusCode(599)) diff --git a/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt b/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt index f2b4ea503..1239204e9 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogCaptureV1Test.kt @@ -416,6 +416,37 @@ internal class PostHogCaptureV1Test { assertTrue(result.retryIds.isEmpty()) } + @Test + fun `a 404 warns once with the capture URL`() { + val http = httpServers.mockHttp(total = 2, response = MockResponse().setResponseCode(404)) + val host = http.url("/").toString().trimEnd('/') + val sut = api(host) + + repeat(2) { assertThrows(PostHogApiError::class.java) { sut.captureV1(listOf(event())) } } + + assertEquals( + listOf( + "$host/i/v1/analytics/events returned 404. Events stay queued and are retried. " + + "If you use a reverse proxy or self-host PostHog, make sure it serves /i/v1/analytics/events.", + ), + logger.warnings, + ) + } + + @Test + fun `the 404 warning leaves out credentials and the query in the host`() { + val http = httpServers.mockHttp(response = MockResponse().setResponseCode(404)) + val origin = http.url("/").toString().trimEnd('/') + val host = origin.replace("://", "://proxy-user:proxy-password@") + "?secret=s3cr3t" + val sut = api(host) + + assertThrows(PostHogApiError::class.java) { sut.captureV1(listOf(event())) } + + val warning = logger.warnings.single() + assertTrue(warning.startsWith("$origin/"), warning) + listOf("proxy-user", "proxy-password", "s3cr3t").forEach { assertFalse(warning.contains(it), warning) } + } + @Test fun `a 503 with Retry-After 0 is not resent by the HTTP client`() { val http = httpServers.mockHttp(response = MockResponse().setResponseCode(503).setHeader("Retry-After", "0")) @@ -706,9 +737,9 @@ internal class PostHogCaptureV1Test { } @Test - fun `queue drops the batch on 429 and keeps it on 503`() { - val http = httpServers.mockHttp(response = MockResponse().setResponseCode(429)) - http.enqueue(MockResponse().setResponseCode(503)) + fun `queue drops the batch on 400 and keeps it on 429`() { + val http = httpServers.mockHttp(response = MockResponse().setResponseCode(400)) + http.enqueue(MockResponse().setResponseCode(429)) val sut = queue(http) sut.add(event()) diff --git a/posthog/src/test/java/com/posthog/internal/PostHogQueueTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogQueueTest.kt index 9ad84b2e7..8e007ccc4 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogQueueTest.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogQueueTest.kt @@ -990,12 +990,14 @@ internal class PostHogQueueTest { } @Test - fun `does not retry on 429`() { - val e = PostHogApiError(429, "", null) + fun `retries on 404 and 429`() { val config = PostHogConfig(PROJECT_TOKEN) val limits = initialBatchLimits(config) - assertTrue(deleteFilesIfAPIError(e, limits, actualBatchSize = limits.cap, logger = config.logger)) + for (status in listOf(404, 429)) { + val e = PostHogApiError(status, "", null) + assertFalse(deleteFilesIfAPIError(e, limits, actualBatchSize = limits.cap, logger = config.logger)) + } } @Test From 739822ba489584c405ac261c87bb45511f6b8f96 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 28 Oct 2026 16:15:50 -0400 Subject: [PATCH 17/19] docs: escape dollar signs in captureAi examples Co-Authored-By: Claude Opus 5.5 --- docs/migration-3.x-to-4.0.md | 5 ++++- posthog/src/main/java/com/posthog/PostHogInterface.kt | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index daf5b3bee..52d1021a1 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -158,7 +158,10 @@ When the event is sent, the SDK removes each one from `properties` and uses it f It takes the same arguments as `capture`, and the events go through the same `beforeSend` hooks. ```kotlin -PostHog.captureAi("$ai_generation", properties = mapOf("$ai_model" to "gpt-5", "$ai_trace_id" to traceId)) +PostHog.captureAi( + "\$ai_generation", + properties = mapOf("\$ai_model" to "gpt-5", "\$ai_trace_id" to traceId), +) ``` - `captureAi` returns the event's uuid, or `null` when the event isn't queued: the SDK is disabled or opted out, `beforeSend` dropped it, or it's over the size limit. diff --git a/posthog/src/main/java/com/posthog/PostHogInterface.kt b/posthog/src/main/java/com/posthog/PostHogInterface.kt index 9cc064879..2da6f24cc 100644 --- a/posthog/src/main/java/com/posthog/PostHogInterface.kt +++ b/posthog/src/main/java/com/posthog/PostHogInterface.kt @@ -125,7 +125,10 @@ public interface PostHogInterface : PostHogCoreInterface { * * Example: * ``` - * PostHog.captureAi("$ai_generation", properties = mapOf("$ai_model" to "gpt-5", "$ai_trace_id" to traceId)) + * PostHog.captureAi( + * "\$ai_generation", + * properties = mapOf("\$ai_model" to "gpt-5", "\$ai_trace_id" to traceId), + * ) * ``` * @param event the AI event name, such as `$ai_generation` * @param distinctId the distinctId, the generated [distinctId] is used if not given or blank From ed59002e73fe1141ca3cab0cead8dda4ad128c69 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 28 Oct 2026 16:15:50 -0400 Subject: [PATCH 18/19] test: give the serializer privacy fixture a prompt-bearing toString Co-Authored-By: Claude Opus 5.5 --- .../java/com/posthog/internal/PostHogSerializerTest.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt index 256704b5d..287c2fc62 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogSerializerTest.kt @@ -315,8 +315,11 @@ internal class PostHogSerializerTest { assertEquals(expectedJson, actualJson) } - // not a data class: its generated hashCode calls Double.hashCode(double), missing on API 23 - private class Turn(val prompt: String, val score: Double) + // not a data class: its generated hashCode calls Double.hashCode(double), missing on API 23. + // toString carries the prompt so logging the value would leak it + private class Turn(val prompt: String, val score: Double) { + override fun toString(): String = "Turn(prompt=$prompt)" + } @Test fun `an unserializable property is logged by key and type, never by value`() { From 83e9b67ddb545574754a839edd957f06fcff8998 Mon Sep 17 00:00:00 2001 From: Anna Garcia Date: Wed, 28 Oct 2026 16:36:34 -0400 Subject: [PATCH 19/19] fix: make captureAiMaxEventBytes limit the whole serialized event Match posthog-ios, posthog-python and the Flutter desktop lane: the value is the serialized event size, defaulting to 8 MiB + 64 KiB, instead of a properties size the SDK added 64 KiB to. A value over the ceiling now logs a warning when it is clamped. Co-Authored-By: Claude Opus 5.5 --- .changeset/android-v4-major.md | 2 +- docs/migration-3.x-to-4.0.md | 2 +- posthog/src/main/java/com/posthog/PostHog.kt | 7 +++++++ .../main/java/com/posthog/PostHogConfig.kt | 16 ++++++++-------- .../main/java/com/posthog/PostHogInterface.kt | 4 ++-- .../java/com/posthog/internal/EndpointSpec.kt | 15 ++++----------- .../java/com/posthog/PostHogCaptureAiTest.kt | 19 +++++++++++++++---- .../internal/PostHogQueueByteLimitsTest.kt | 2 +- 8 files changed, 39 insertions(+), 28 deletions(-) diff --git a/.changeset/android-v4-major.md b/.changeset/android-v4-major.md index d6b6ab1c1..5e03a0731 100644 --- a/.changeset/android-v4-major.md +++ b/.changeset/android-v4-major.md @@ -30,5 +30,5 @@ - **Breaking:** skip deep link query parameters whose name starts with `$` in the `Deep Link Opened` event, so a link can't set a PostHog property such as `$process_person_profile` - stop storing or matching the person-properties deduplication hash for a `$set` sent with `process_person_profile: false` - debug logs for capture requests name the event count and the response status instead of printing the request and response bodies, and name each event PostHog drops with the reason it gives -- add `captureAiCompression`, `captureAiMaxQueueSize`, `captureAiTimeout` and `captureAiMaxEventBytes` to `PostHogConfig` to tune AI requests separately from other events; the defaults are gzip, 100 queued events (and at most 50 MiB), 60 seconds and 8 MiB +- add `captureAiCompression`, `captureAiMaxQueueSize`, `captureAiTimeout` and `captureAiMaxEventBytes` to `PostHogConfig` to tune AI requests separately from other events; the defaults are gzip, 100 queued events (and at most 50 MiB), 60 seconds and 8 MiB + 64 KiB per serialized event - cap a request at about 5 MiB of events, for analytics and AI events, and send as soon as that much is queued; an event larger than that is sent on its own diff --git a/docs/migration-3.x-to-4.0.md b/docs/migration-3.x-to-4.0.md index 52d1021a1..ab5a49ba7 100644 --- a/docs/migration-3.x-to-4.0.md +++ b/docs/migration-3.x-to-4.0.md @@ -165,7 +165,7 @@ PostHog.captureAi( ``` - `captureAi` returns the event's uuid, or `null` when the event isn't queued: the SDK is disabled or opted out, `beforeSend` dropped it, or it's over the size limit. -- An AI event's properties can be up to 8 MiB. The SDK drops a serialized event over 8 MiB + 64 KiB, measured before `encryption`, and logs the event name and size, never its content. Set `captureAiMaxEventBytes` to lower the properties limit; the SDK adds 64 KiB to it for the rest of the event. +- An AI event's properties can be up to 8 MiB. The SDK drops a serialized event over 8 MiB + 64 KiB, measured before `encryption`, and logs the event name and size, never its content. Set `captureAiMaxEventBytes` to lower that limit; a larger value uses 8 MiB + 64 KiB and logs a warning. - PostHog drops an event sent with `captureAi` whose name doesn't start with `$ai_`. `capture` never sends events to the AI endpoint, even ones named `$ai_*`. - AI requests time out after 60 seconds without network activity, or `captureAiTimeout`. Other requests keep OkHttp's 10-second defaults. - AI events are queued on disk separately from other events (`posthog-disk-ai-queue` in the app's cache directory). Up to 100 events (`captureAiMaxQueueSize`) and 50 MiB are kept, and the oldest are dropped first. `maxQueueSize` doesn't apply. diff --git a/posthog/src/main/java/com/posthog/PostHog.kt b/posthog/src/main/java/com/posthog/PostHog.kt index 97eff37e3..3edda9e0b 100644 --- a/posthog/src/main/java/com/posthog/PostHog.kt +++ b/posthog/src/main/java/com/posthog/PostHog.kt @@ -270,6 +270,13 @@ public class PostHog private constructor( EndpointSpec.logs(config, api, config.logsStoragePrefix), logsExecutor, ) + val aiEventBytesCeiling = PostHogConfig.DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES + if (config.captureAiMaxEventBytes > aiEventBytesCeiling) { + config.logger.log( + "captureAiMaxEventBytes ${config.captureAiMaxEventBytes} is over the " + + "$aiEventBytesCeiling byte ceiling, using $aiEventBytesCeiling.", + ) + } val aiQueue = PostHogQueue( config, diff --git a/posthog/src/main/java/com/posthog/PostHogConfig.kt b/posthog/src/main/java/com/posthog/PostHogConfig.kt index 188f56283..c2977be9f 100644 --- a/posthog/src/main/java/com/posthog/PostHogConfig.kt +++ b/posthog/src/main/java/com/posthog/PostHogConfig.kt @@ -454,14 +454,14 @@ public open class PostHogConfig( public var captureAiTimeout: Int = DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS /** - * Maximum size, in bytes, of a [PostHog.captureAi] event's properties. The SDK drops a serialized - * event over this plus 64 KiB for the rest of the event, measured before [encryption]; then - * `captureAi` returns null and logs only the event name and size. + * Maximum size, in bytes, of a serialized [PostHog.captureAi] event, measured before [encryption]. + * The SDK drops a larger event; then `captureAi` returns null and logs only the event name and size. * - * Use it to lower the limit. PostHog drops an AI event whose properties are over 8 MiB, so a - * larger value, or a value of 0 or less, uses 8 MiB. + * Use it to lower the limit. PostHog drops AI events whose properties are over 8 MiB, so the default + * allows 8 MiB plus 64 KiB for the rest of the event. A larger value uses the default and logs a + * warning. A value of 0 or less uses the default. * - * Default: 8388608 (8 MiB). + * Default: 8454144 (8 MiB + 64 KiB). */ public var captureAiMaxEventBytes: Int = DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES @@ -642,8 +642,8 @@ public open class PostHogConfig( /** Default for [captureAiTimeout]. */ public const val DEFAULT_CAPTURE_AI_TIMEOUT_SECONDS: Int = 60 - /** Default for [captureAiMaxEventBytes]: PostHog's 8 MiB limit on an AI event's properties. */ - public const val DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES: Int = 8 * 1024 * 1024 + /** Default for [captureAiMaxEventBytes]: PostHog's 8 MiB limit on an AI event's properties, plus 64 KiB for the rest of the event. */ + public const val DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES: Int = 8 * 1024 * 1024 + 64 * 1024 public const val DEFAULT_US_HOST: String = "https://us.i.posthog.com" public const val DEFAULT_US_ASSETS_HOST: String = "https://us-assets.i.posthog.com" diff --git a/posthog/src/main/java/com/posthog/PostHogInterface.kt b/posthog/src/main/java/com/posthog/PostHogInterface.kt index 2da6f24cc..eaa993bf4 100644 --- a/posthog/src/main/java/com/posthog/PostHogInterface.kt +++ b/posthog/src/main/java/com/posthog/PostHogInterface.kt @@ -118,8 +118,8 @@ public interface PostHogInterface : PostHogCoreInterface { * [PostHogConfig.flushIntervalSeconds], like other events, and * [PostHogConfig.captureAiCompression], [PostHogConfig.captureAiMaxQueueSize] and * [PostHogConfig.captureAiTimeout] instead of `compression`, `maxQueueSize` and the default - * timeouts. A serialized event over [PostHogConfig.captureAiMaxEventBytes] plus 64 KiB (8 MiB + - * 64 KiB by default) is dropped, with a log line naming the event and its size. Requests carry up to about + * timeouts. A serialized event over [PostHogConfig.captureAiMaxEventBytes] (8 MiB + 64 KiB by + * default) is dropped, with a log line naming the event and its size. Requests carry up to about * 5 MiB of events, and the SDK sends as soon as that much is queued. Up to 50 MiB of AI events * are kept on disk; beyond that, the oldest are dropped. * diff --git a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt index 41abd466f..a628c87e0 100644 --- a/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt +++ b/posthog/src/main/java/com/posthog/internal/EndpointSpec.kt @@ -79,7 +79,7 @@ public class EndpointSpec internal constructor( * Returns the [EndpointSpec] for the capture v1 AI endpoint (`/i/v1/ai/events`), used by * `captureAi`. Same codec, results and retry policy as [batch], on its own queue: AI events * can be megabytes, so the queue is capped by bytes as well as count, and an event over - * [PostHogConfig.captureAiMaxEventBytes] (plus the stored envelope) is dropped when queued. + * [PostHogConfig.captureAiMaxEventBytes] is dropped when queued. */ @JvmStatic public fun ai( @@ -206,12 +206,6 @@ internal const val CAPTURE_V1_MAX_BATCH_BYTES: Long = 5L * 1024 * 1024 /** Disk budget for queued AI events; the oldest are dropped first. */ internal const val AI_MAX_QUEUE_BYTES: Long = 50L * 1024 * 1024 -/** - * The stored AI event also holds its envelope (name, ids, timestamp), so this much is allowed on - * top of the properties limit, which PostHog measures on the properties alone. - */ -internal const val AI_EVENT_ENVELOPE_BYTES: Long = 64L * 1024 - /** [PostHogConfig.captureAiMaxQueueSize], or its default when it isn't positive. */ internal val PostHogConfig.effectiveCaptureAiMaxQueueSize: Int get() = if (captureAiMaxQueueSize > 0) captureAiMaxQueueSize else PostHogConfig.DEFAULT_CAPTURE_AI_MAX_QUEUE_SIZE @@ -260,14 +254,13 @@ internal fun PostHogConfig.serializedSize( } /** - * Size limit for a serialized AI event, before any [PostHogConfig.encryption]: [PostHogConfig.captureAiMaxEventBytes], capped at PostHog's - * 8 MiB limit, plus the envelope allowance. + * Size limit for a serialized AI event, before any [PostHogConfig.encryption]: [PostHogConfig.captureAiMaxEventBytes], + * capped at its default (PostHog's 8 MiB properties limit plus 64 KiB for the rest of the event). */ internal val PostHogConfig.captureAiMaxStoredEventBytes: Long get() { val limit = PostHogConfig.DEFAULT_CAPTURE_AI_MAX_EVENT_BYTES - val bytes = if (captureAiMaxEventBytes > 0) minOf(captureAiMaxEventBytes, limit) else limit - return bytes + AI_EVENT_ENVELOPE_BYTES + return (if (captureAiMaxEventBytes > 0) minOf(captureAiMaxEventBytes, limit) else limit).toLong() } /** `/snapshot` (replay) retry policy: 408, 429, the listed 5xx, plus 3xx redirects. */ diff --git a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt index 13b89599b..938316749 100644 --- a/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogCaptureAiTest.kt @@ -183,7 +183,6 @@ internal class PostHogCaptureAiTest { @Test fun `an AI event over captureAiMaxEventBytes returns null and logs only its name and size`() { val http = httpServers.mockHttp() - // the limit is the configured bytes plus a 64 KiB envelope val sut = getSut(http) { captureAiMaxEventBytes = 1 } val prompt = "p".repeat(70_000) @@ -197,12 +196,24 @@ internal class PostHogCaptureAiTest { assertTrue(aiQueueFiles().isEmpty()) val dropped = logger.messages.single { it.startsWith("Dropping AI event") } assertTrue( - dropped.matches(Regex("Dropping AI event \\\$ai_generation of more than ${1 + 64 * 1024} bytes, over the limit\\.")), + dropped.matches(Regex("Dropping AI event \\\$ai_generation of more than 1 bytes, over the limit\\.")), dropped, ) assertTrue(logger.messages.none { it.contains("pppp") }) } + @Test + fun `a captureAiMaxEventBytes over the ceiling logs a warning`() { + getSut(httpServers.mockHttp()) { captureAiMaxEventBytes = 16 * 1024 * 1024 } + + assertTrue( + logger.messages.any { + it == "captureAiMaxEventBytes 16777216 is over the 8454144 byte ceiling, using 8454144." + }, + logger.messages.toString(), + ) + } + @Test fun `an AI event whose properties are exactly 8 MiB is sent`() { val ceiling = 8 * 1024 * 1024 @@ -230,7 +241,7 @@ internal class PostHogCaptureAiTest { val http = httpServers.mockHttp() val sut = getSut(http) { - captureAiMaxEventBytes = 1 + captureAiMaxEventBytes = 100_000 // writes every byte twice, so the stored event is double its JSON size encryption = object : PostHogEncryption { @@ -275,7 +286,7 @@ internal class PostHogCaptureAiTest { } } } - // under the 64 KiB + 1 byte limit as JSON, over it once encrypted + // under the 100,000 byte limit as JSON, over it once encrypted val prompt = "p".repeat(60_000) val uuid = sut.captureAi("\$ai_generation", properties = mapOf("\$ai_input" to prompt)) diff --git a/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt b/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt index d89344426..574c26c7c 100644 --- a/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt +++ b/posthog/src/test/java/com/posthog/internal/PostHogQueueByteLimitsTest.kt @@ -191,7 +191,7 @@ internal class PostHogQueueByteLimitsTest { val api = PostHogApi(config) config.captureAiMaxEventBytes = 1024 - assertEquals(1024L + 64 * 1024, config.captureAiMaxStoredEventBytes) + assertEquals(1024L, config.captureAiMaxStoredEventBytes) config.captureAiMaxEventBytes = 16 * 1024 * 1024 assertEquals(8L * 1024 * 1024 + 64 * 1024, config.captureAiMaxStoredEventBytes) config.captureAiMaxEventBytes = 0