diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_2_dlq_replay.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_2_dlq_replay.yaml new file mode 100644 index 0000000000..0cd0ef2335 --- /dev/null +++ b/snuba/datasets/configuration/events_analytics_platform/storages/eap_items_2_dlq_replay.yaml @@ -0,0 +1,158 @@ +version: v1 +kind: writable_storage +name: eap_items_2_dlq_replay + +storage: + key: eap_items_2_dlq_replay + set_key: events_analytics_platform + +readiness_state: complete + +schema: + columns: + [ + { name: organization_id, type: UInt, args: { size: 64 } }, + { name: project_id, type: UInt, args: { size: 64 } }, + { name: item_type, type: UInt, args: { size: 8 } }, + { name: timestamp, type: DateTime }, + { name: trace_id, type: UUID }, + { name: conversation_id, type: UUID }, + { name: session_id, type: UUID }, + { name: ai_conversation_id, type: String }, + { name: item_id, type: UInt, args: { size: 128 } }, + { name: sampling_weight, type: UInt, args: { size: 64 } }, + { name: sampling_factor, type: Float, args: { size: 64 } }, + { name: retention_days, type: UInt, args: { size: 16 } }, + { name: indexed_name, type: String }, + { name: downsampled_retention_days, type: UInt, args: { size: 16 } }, + + { name: attributes_bool, type: Map, args: { key: { type: String }, value: { type: Bool } } }, + { name: attributes_int, type: Map, args: { key: { type: String }, value: { type: Int, args: { size: 64 } } } }, + + { name: attributes_string_0, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_1, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_2, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_3, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_4, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_5, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_6, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_7, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_8, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_9, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_10, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_11, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_12, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_13, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_14, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_15, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_16, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_17, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_18, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_19, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_20, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_21, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_22, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_23, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_24, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_25, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_26, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_27, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_28, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_29, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_30, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_31, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_32, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_33, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_34, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_35, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_36, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_37, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_38, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_string_39, type: Map, args: { key: { type: String }, value: { type: String } } }, + { name: attributes_float_0, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_1, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_2, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_3, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_4, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_5, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_6, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_7, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_8, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_9, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_10, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_11, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_12, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_13, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_14, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_15, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_16, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_17, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_18, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_19, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_20, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_21, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_22, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_23, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_24, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_25, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_26, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_27, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_28, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_29, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_30, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_31, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_32, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_33, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_34, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_35, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_36, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_37, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_38, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + { name: attributes_float_39, type: Map, args: { key: { type: String }, value: { type: Float, args: { size: 64 } } } }, + + { name: attributes_array_string, type: Map, args: { key: { type: String }, value: { type: Array, args: { inner_type: { type: String } } } } }, + { name: attributes_array_int, type: Map, args: { key: { type: String }, value: { type: Array, args: { inner_type: { type: Int, args: { size: 64 } } } } } }, + { name: attributes_array_float, type: Map, args: { key: { type: String }, value: { type: Array, args: { inner_type: { type: Float, args: { size: 64 } } } } } }, + { name: attributes_array_bool, type: Map, args: { key: { type: String }, value: { type: Array, args: { inner_type: { type: Bool } } } } }, + ] + local_table_name: eap_items_2_local + dist_table_name: eap_items_2_dist + partition_format: [retention_days, date] + +query_processors: + - processor: UniqInSelectAndHavingProcessor + - processor: UUIDColumnProcessor + args: + columns: [trace_id, conversation_id, session_id] + - processor: HexIntColumnProcessor + args: + columns: [item_id] + size: 32 + - processor: PrewhereProcessor + args: + prewhere_candidates: + [trace_id] + - processor: TupleUnaliaser + - processor: ClickhouseSettingsOverride + args: + overwrite_existing: false + settings: + max_execution_time: 25 + timeout_before_checking_execution_speed: 0 + distributed_product_mode: global + +mandatory_condition_checkers: + - condition: OrgIdEnforcer + args: + field_name: organization_id + +# Consumes from the eap_items DLQ topic and writes to the same ClickHouse +# tables as the primary `eap_items_2` storage. Same role as +# `eap_items_dlq_replay`, but for regions that have cut over to +# eap_items_2. Intentionally has no `dlq_topic`: messages that fail +# processing here will fail the consumer loudly rather than getting silently +# re-DLQ'd back into the same topic. It also has its own storage name, so +# `eap_items_dlq_grace_period_min` entries for `eap_items_2` do not apply. +stream_loader: + processor: EAPItemsProcessor + default_topic: snuba-dead-letter-items