From 53481c99598314093c5f0f957032c36b22eba23c Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Wed, 29 Apr 2026 09:32:47 +0700 Subject: [PATCH 1/7] fix: add more things to inspect failures --- action.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/action.yaml b/action.yaml index 601aaa13099..3877efc5bdb 100644 --- a/action.yaml +++ b/action.yaml @@ -264,3 +264,9 @@ runs: echo "::group::Inspect failure - docker compose logs" docker compose logs echo "::endgroup::" + echo "::group::Inspect failure - docker stats" + docker stats --no-stream + echo "::endgroup::" + echo "::group::Inspect failure - disk i/o stats" + iostat -x 1 + echo "::endgroup::" From 7abcbcdd049b79286d132962cb320ff5235e3297 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Wed, 29 Apr 2026 14:45:28 +0700 Subject: [PATCH 2/7] fix(ci): declare timeout minutes --- .github/workflows/test.yml | 2 ++ action.yaml | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e24bdca5251..4c6e02d4dc2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,7 @@ jobs: unit-test: if: github.repository_owner == 'getsentry' runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: matrix: os: [ubuntu-24.04, ubuntu-24.04-arm] @@ -40,6 +41,7 @@ jobs: integration-test: if: github.repository_owner == 'getsentry' runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: diff --git a/action.yaml b/action.yaml index 3877efc5bdb..5846f1f4f28 100644 --- a/action.yaml +++ b/action.yaml @@ -267,6 +267,3 @@ runs: echo "::group::Inspect failure - docker stats" docker stats --no-stream echo "::endgroup::" - echo "::group::Inspect failure - disk i/o stats" - iostat -x 1 - echo "::endgroup::" From 3cf271a560474c3f5a09d81326489e09ef5d0dcd Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Tue, 5 May 2026 11:09:43 +0700 Subject: [PATCH 3/7] feat: longer kafka socket timeout duration --- sentry/sentry.conf.example.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 3bdc3e1a19d..4af5e13744b 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -221,7 +221,12 @@ def get_internal_network(): DEFAULT_KAFKA_OPTIONS = { "bootstrap.servers": "kafka:9092", "message.max.bytes": 50000000, - "socket.timeout.ms": 1000, + "socket.timeout.ms": 10000, + "request.timeout.ms": 30000, + "retries": 5, + "retry.backoff.ms": 1000, + "reconnect.backoff.ms": 1000, + "reconnect.backoff.max.ms": 10000, } SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" From ebf12465ecd909a7be1c865ec826cdd0c1d5db76 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Tue, 5 May 2026 21:27:08 +0700 Subject: [PATCH 4/7] feat: tune kafka to be more lenient --- docker-compose.yml | 4 ++-- sentry/sentry.conf.example.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b5d768f8ca0..d67949aa444 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -196,8 +196,8 @@ services: KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" ulimits: nofile: - soft: 4096 - hard: 4096 + soft: 100000 + hard: 100000 volumes: - "sentry-kafka:/var/lib/kafka/data" - "sentry-kafka-log:/var/lib/kafka/log" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 4af5e13744b..b901986ef9a 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -221,12 +221,17 @@ def get_internal_network(): DEFAULT_KAFKA_OPTIONS = { "bootstrap.servers": "kafka:9092", "message.max.bytes": 50000000, - "socket.timeout.ms": 10000, - "request.timeout.ms": 30000, - "retries": 5, - "retry.backoff.ms": 1000, - "reconnect.backoff.ms": 1000, - "reconnect.backoff.max.ms": 10000, + "socket.timeout.ms": 10000, # Timeout for individual socket operations (send/recv) + "request.timeout.ms": 30000, # Max time to wait for a broker response before failing + "retries": 5, # Number of retries for transient/retriable request failures + "retry.backoff.ms": 1000, # Wait time between retry attempts + "reconnect.backoff.ms": 1000, # Initial wait before reconnecting after a lost connection + "reconnect.backoff.max.ms": 10000, # Upper bound for exponential backoff on reconnect attempts + # Session & heartbeat — must satisfy: + # heartbeat.interval.ms < session.timeout.ms < max.poll.interval.ms + "session.timeout.ms": 60000, # Grace period before broker evicts an unresponsive consumer (default: 45s) + "heartbeat.interval.ms": 20000, # How often the consumer sends a heartbeat — must be 1/3 of session.timeout.ms + "max.poll.interval.ms": 600000, # Max allowed time between poll() calls before the consumer is considered dead } SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" From f9ce3cc5e697e814e013f5c1c0e17f0bb0284354 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Fri, 22 May 2026 11:23:04 +0700 Subject: [PATCH 5/7] chore: increase kafka request timeout to 60s --- sentry/sentry.conf.example.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 2efd08329b8..b0a081198bc 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -221,17 +221,17 @@ def get_internal_network(): DEFAULT_KAFKA_OPTIONS = { "bootstrap.servers": "kafka:9092", "message.max.bytes": 50000000, - "socket.timeout.ms": 10000, # Timeout for individual socket operations (send/recv) - "request.timeout.ms": 30000, # Max time to wait for a broker response before failing + "socket.timeout.ms": 10_000, # Timeout for individual socket operations (send/recv) + "request.timeout.ms": 60_000, # Max time to wait for a broker response before failing "retries": 5, # Number of retries for transient/retriable request failures "retry.backoff.ms": 1000, # Wait time between retry attempts "reconnect.backoff.ms": 1000, # Initial wait before reconnecting after a lost connection - "reconnect.backoff.max.ms": 10000, # Upper bound for exponential backoff on reconnect attempts + "reconnect.backoff.max.ms": 10_000, # Upper bound for exponential backoff on reconnect attempts # Session & heartbeat — must satisfy: # heartbeat.interval.ms < session.timeout.ms < max.poll.interval.ms - "session.timeout.ms": 60000, # Grace period before broker evicts an unresponsive consumer (default: 45s) - "heartbeat.interval.ms": 20000, # How often the consumer sends a heartbeat — must be 1/3 of session.timeout.ms - "max.poll.interval.ms": 600000, # Max allowed time between poll() calls before the consumer is considered dead + "heartbeat.interval.ms": 20_000, # How often the consumer sends a heartbeat — must be 1/3 of session.timeout.ms + "session.timeout.ms": 60_000, # Grace period before broker evicts an unresponsive consumer (default: 45s) + "max.poll.interval.ms": 600_000, # Max allowed time between poll() calls before the consumer is considered dead } SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" From 4064d5f1ae456fe27008416ad3ed961f8548f9e6 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 23 May 2026 08:13:50 +0700 Subject: [PATCH 6/7] fix: use --max-poll-interval-ms flag --- .env | 7 +++ docker-compose.yml | 96 +++++++++++++++++------------------ sentry/sentry.conf.example.py | 12 +---- 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/.env b/.env index 483f3748751..5ebd7995c6f 100644 --- a/.env +++ b/.env @@ -43,3 +43,10 @@ HEALTHCHECK_FILE_START_PERIOD=600s # It's also the proper way to monitor self-hosted Sentry systems. # Set STATSD_ADDR to a valid IP:PORT combination to enable Statsd metrics collection. # STATSD_ADDR=127.0.0.1:8125 + +# Set SENTRY_KAFKA_MAX_POLL_INTERVAL_MS to a value (in milliseconds) that is +# higher than the expected maximum processing time of a batch of messages for +# the Kafka consumers. This will prevent Kafka from considering the consumer +# as failed and triggering a rebalance while it is still processing messages. +# The default value is 30000 ms (30 seconds). +# SENTRY_KAFKA_MAX_POLL_INTERVAL_MS=30000 diff --git a/docker-compose.yml b/docker-compose.yml index 2b1061c0efb..681d08eac67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -337,7 +337,7 @@ services: # Kafka consumer responsible for feeding events into Clickhouse snuba-errors-consumer: <<: *snuba_defaults - command: rust-consumer --storage errors --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage errors --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults # Kafka consumer responsible for feeding outcomes into Clickhouse @@ -345,17 +345,17 @@ services: # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults - command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults snuba-outcomes-billing-consumer: <<: *snuba_defaults - command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --raw-events-topic outcomes-billing --health-check-file /tmp/health.txt + command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --raw-events-topic outcomes-billing --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults snuba-group-attributes-consumer: <<: *snuba_defaults - command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults snuba-replacer: @@ -363,7 +363,7 @@ services: command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset snuba-subscription-consumer-events: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults ############################################# @@ -372,133 +372,133 @@ services: # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults - command: rust-consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-replays-consumer: <<: *snuba_defaults - command: rust-consumer --storage replays --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage replays --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-issue-occurrence-consumer: <<: *snuba_defaults - command: rust-consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-metrics-consumer: <<: *snuba_defaults - command: rust-consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-transactions: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-metrics: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-distributions: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-sets: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-counters: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-gauges: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-distributions-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-sets-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-counters-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-gauges-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-profiling-profiles-consumer: <<: *snuba_defaults - command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-profiling-functions-consumer: <<: *snuba_defaults - command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-profiling-profile-chunks-consumer: <<: *snuba_defaults - command: rust-consumer --storage profile_chunks --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + command: rust-consumer --storage profile_chunks --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-eap-items-consumer: <<: *snuba_defaults - command: rust-consumer --storage eap_items --consumer-group eap_items_group --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --use-rust-processor --health-check-file /tmp/health.txt + command: rust-consumer --storage eap_items --consumer-group eap_items_group --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --use-rust-processor --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-eap-items: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults symbolicator: @@ -544,22 +544,22 @@ services: - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' events-consumer: <<: *sentry_defaults - command: run consumer ingest-events --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-events --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults attachments-consumer: <<: *sentry_defaults - command: run consumer ingest-attachments --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-attachments --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults post-process-forwarder-errors: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers --healthcheck-file-path /tmp/health.txt + command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults subscription-consumer-events: <<: *sentry_defaults - command: run consumer events-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer events-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults ############################################## @@ -567,140 +567,140 @@ services: ############################################## transactions-consumer: <<: *sentry_defaults - command: run consumer ingest-transactions --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-transactions --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete metrics-consumer: <<: *sentry_defaults - command: run consumer ingest-metrics --consumer-group metrics-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-metrics --consumer-group metrics-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete generic-metrics-consumer: <<: *sentry_defaults - command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete billing-metrics-consumer: <<: *sentry_defaults - command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete ingest-replay-recordings: <<: *sentry_defaults - command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete ingest-occurrences: <<: *sentry_defaults - command: run consumer ingest-occurrences --consumer-group ingest-occurrences --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-occurrences --consumer-group ingest-occurrences --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete ingest-profiles: <<: *sentry_defaults - command: run consumer ingest-profiles --consumer-group ingest-profiles --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-profiles --consumer-group ingest-profiles --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete ingest-monitors: <<: *sentry_defaults - command: run consumer ingest-monitors --consumer-group ingest-monitors --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-monitors --consumer-group ingest-monitors --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete ingest-feedback-events: <<: *sentry_defaults - command: run consumer ingest-feedback-events --consumer-group ingest-feedback --healthcheck-file-path /tmp/health.txt + command: run consumer ingest-feedback-events --consumer-group ingest-feedback --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete process-spans: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset process-spans --consumer-group process-spans --healthcheck-file-path /tmp/health.txt + command: run consumer --no-strict-offset-reset process-spans --consumer-group process-spans --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete process-segments: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset process-segments --consumer-group process-segments --healthcheck-file-path /tmp/health.txt + command: run consumer --no-strict-offset-reset process-segments --consumer-group process-segments --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete monitors-clock-tick: <<: *sentry_defaults - command: run consumer monitors-clock-tick --consumer-group monitors-clock-tick --healthcheck-file-path /tmp/health.txt + command: run consumer monitors-clock-tick --consumer-group monitors-clock-tick --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete monitors-clock-tasks: <<: *sentry_defaults - command: run consumer monitors-clock-tasks --consumer-group monitors-clock-tasks --healthcheck-file-path /tmp/health.txt + command: run consumer monitors-clock-tasks --consumer-group monitors-clock-tasks --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete uptime-results: <<: *sentry_defaults - command: run consumer uptime-results --consumer-group uptime-results --healthcheck-file-path /tmp/health.txt + command: run consumer uptime-results --consumer-group uptime-results --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete post-process-forwarder-transactions: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group --healthcheck-file-path /tmp/health.txt + command: run consumer --no-strict-offset-reset post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete post-process-forwarder-issue-platform: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group --healthcheck-file-path /tmp/health.txt + command: run consumer --no-strict-offset-reset post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-transactions: <<: *sentry_defaults - command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-eap-items: <<: *sentry_defaults - command: run consumer subscription-results-eap-items --consumer-group subscription-results-eap-items --healthcheck-file-path /tmp/health.txt + command: run consumer subscription-results-eap-items --consumer-group subscription-results-eap-items --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-metrics: <<: *sentry_defaults - command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-generic-metrics: <<: *sentry_defaults - command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} healthcheck: <<: *file_healthcheck_defaults profiles: diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index b0a081198bc..842cc1daa1e 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -221,17 +221,7 @@ def get_internal_network(): DEFAULT_KAFKA_OPTIONS = { "bootstrap.servers": "kafka:9092", "message.max.bytes": 50000000, - "socket.timeout.ms": 10_000, # Timeout for individual socket operations (send/recv) - "request.timeout.ms": 60_000, # Max time to wait for a broker response before failing - "retries": 5, # Number of retries for transient/retriable request failures - "retry.backoff.ms": 1000, # Wait time between retry attempts - "reconnect.backoff.ms": 1000, # Initial wait before reconnecting after a lost connection - "reconnect.backoff.max.ms": 10_000, # Upper bound for exponential backoff on reconnect attempts - # Session & heartbeat — must satisfy: - # heartbeat.interval.ms < session.timeout.ms < max.poll.interval.ms - "heartbeat.interval.ms": 20_000, # How often the consumer sends a heartbeat — must be 1/3 of session.timeout.ms - "session.timeout.ms": 60_000, # Grace period before broker evicts an unresponsive consumer (default: 45s) - "max.poll.interval.ms": 600_000, # Max allowed time between poll() calls before the consumer is considered dead + "socket.timeout.ms": 1000, } SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" From 8b515125a5f65c597f7db998c309817f2c039929 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 23 May 2026 13:28:59 +0700 Subject: [PATCH 7/7] fix: remove --max-poll-interval-ms on snuba subscriptions consumers --- docker-compose.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 681d08eac67..ef6e9df797a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,6 +77,7 @@ x-sentry-defaults: &sentry_defaults SENTRY_SYSTEM_SECRET_KEY: LAUNCHPAD_RPC_SHARED_SECRET: SENTRY_STATSD_ADDR: "${STATSD_ADDR:-}" + SENTRY_KAFKA_MAX_POLL_INTERVAL_MS: volumes: - "sentry-data:/data" - "./sentry:/etc/sentry" @@ -105,6 +106,7 @@ x-snuba-defaults: &snuba_defaults # If you have statsd server, you can utilize that to monitor self-hosted Snuba containers. # To start, state these environment variables below on your `.env.` file and adjust the options as needed. SNUBA_STATSD_ADDR: "${STATSD_ADDR:-}" + SENTRY_KAFKA_MAX_POLL_INTERVAL_MS: services: smtp: <<: *restart_policy @@ -363,7 +365,7 @@ services: command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset snuba-subscription-consumer-events: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults ############################################# @@ -400,42 +402,42 @@ services: - feature-complete snuba-subscription-consumer-transactions: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-metrics: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-distributions: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-sets: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-counters: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-gauges: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults profiles: @@ -498,7 +500,7 @@ services: - feature-complete snuba-subscription-consumer-eap-items: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt --max-poll-interval-ms ${SENTRY_KAFKA_MAX_POLL_INTERVAL_MS:-300000} + command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt healthcheck: <<: *file_healthcheck_defaults symbolicator: