From 3b33064cca6bd05c7576ee1c495cc68e60f93725 Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Fri, 6 Mar 2026 21:24:42 +0530 Subject: [PATCH 1/7] perf: derive overview stats from execution tables, not jobs table Replace COUNT(*) queries on solid_queue_jobs with counts from ready_executions, scheduled_executions, claimed_executions, and failed_executions. Replaces "Total Jobs" and "Completed" dashboard stats with "Active Jobs" (sum of ready + scheduled + in-progress + failed). Resolves gateway timeouts on overview page with millions of rows. Fixes #27 --- .../solid_queue_monitor/stats_presenter.rb | 3 +- .../solid_queue_monitor/stats_calculator.rb | 20 +++++++----- .../stats_presenter_spec.rb | 11 +++---- .../stats_calculator_spec.rb | 32 +++++++++---------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/app/presenters/solid_queue_monitor/stats_presenter.rb b/app/presenters/solid_queue_monitor/stats_presenter.rb index ea85042..9c46002 100644 --- a/app/presenters/solid_queue_monitor/stats_presenter.rb +++ b/app/presenters/solid_queue_monitor/stats_presenter.rb @@ -11,13 +11,12 @@ def render

Queue Statistics

- #{generate_stat_card('Total Jobs', @stats[:total_jobs])} + #{generate_stat_card('Active Jobs', @stats[:active_jobs])} #{generate_stat_card('Ready', @stats[:ready])} #{generate_stat_card('In Progress', @stats[:in_progress])} #{generate_stat_card('Scheduled', @stats[:scheduled])} #{generate_stat_card('Recurring', @stats[:recurring])} #{generate_stat_card('Failed', @stats[:failed])} - #{generate_stat_card('Completed', @stats[:completed])}
HTML diff --git a/app/services/solid_queue_monitor/stats_calculator.rb b/app/services/solid_queue_monitor/stats_calculator.rb index 1aeb017..b7c9aa9 100644 --- a/app/services/solid_queue_monitor/stats_calculator.rb +++ b/app/services/solid_queue_monitor/stats_calculator.rb @@ -3,15 +3,19 @@ module SolidQueueMonitor class StatsCalculator def self.calculate + scheduled = SolidQueue::ScheduledExecution.count + ready = SolidQueue::ReadyExecution.count + failed = SolidQueue::FailedExecution.count + in_progress = SolidQueue::ClaimedExecution.count + recurring = SolidQueue::RecurringTask.count + { - total_jobs: SolidQueue::Job.count, - unique_queues: SolidQueue::Job.distinct.count(:queue_name), - scheduled: SolidQueue::ScheduledExecution.count, - ready: SolidQueue::ReadyExecution.count, - failed: SolidQueue::FailedExecution.count, - in_progress: SolidQueue::ClaimedExecution.count, - completed: SolidQueue::Job.where.not(finished_at: nil).count, - recurring: SolidQueue::RecurringTask.count + active_jobs: ready + scheduled + in_progress + failed, + scheduled: scheduled, + ready: ready, + failed: failed, + in_progress: in_progress, + recurring: recurring } end end diff --git a/spec/presenters/solid_queue_monitor/stats_presenter_spec.rb b/spec/presenters/solid_queue_monitor/stats_presenter_spec.rb index 2d2e486..b440cb6 100644 --- a/spec/presenters/solid_queue_monitor/stats_presenter_spec.rb +++ b/spec/presenters/solid_queue_monitor/stats_presenter_spec.rb @@ -8,13 +8,12 @@ let(:stats) do { - total_jobs: 100, + active_jobs: 75, scheduled: 20, ready: 30, in_progress: 15, recurring: 5, - failed: 10, - completed: 40 + failed: 10 } end @@ -27,8 +26,8 @@ html = subject.render expect(html).to include('Queue Statistics') - expect(html).to include('Total Jobs') - expect(html).to include('100') + expect(html).to include('Active Jobs') + expect(html).to include('75') expect(html).to include('Scheduled') expect(html).to include('20') expect(html).to include('Ready') @@ -39,8 +38,6 @@ expect(html).to include('5') expect(html).to include('Failed') expect(html).to include('10') - expect(html).to include('Completed') - expect(html).to include('40') end end end diff --git a/spec/services/solid_queue_monitor/stats_calculator_spec.rb b/spec/services/solid_queue_monitor/stats_calculator_spec.rb index 1a14180..8214751 100644 --- a/spec/services/solid_queue_monitor/stats_calculator_spec.rb +++ b/spec/services/solid_queue_monitor/stats_calculator_spec.rb @@ -5,45 +5,45 @@ RSpec.describe SolidQueueMonitor::StatsCalculator do describe '.calculate' do before do - # Create some test data - # Note: execution factories also create associated jobs - create_list(:solid_queue_job, 3) - create(:solid_queue_job, :completed) - create(:solid_queue_job, :completed) - create(:solid_queue_job, queue_name: 'high_priority') create(:solid_queue_failed_execution) create(:solid_queue_scheduled_execution) create(:solid_queue_ready_execution) + create(:solid_queue_claimed_execution) end it 'returns a hash with all required statistics' do stats = described_class.calculate - expect(stats).to be_a(Hash) expect(stats).to include( - :total_jobs, - :unique_queues, + :active_jobs, :scheduled, :ready, :failed, - :completed, :in_progress, :recurring ) end - it 'calculates the correct counts' do + it 'calculates the correct counts from execution tables' do stats = described_class.calculate - # 6 explicitly created jobs + 3 jobs created by execution factories = 9 total - expect(stats[:total_jobs]).to eq(9) - expect(stats[:unique_queues]).to eq(2) expect(stats[:scheduled]).to eq(1) expect(stats[:ready]).to eq(1) expect(stats[:failed]).to eq(1) - expect(stats[:completed]).to eq(2) - expect(stats[:in_progress]).to eq(0) + expect(stats[:in_progress]).to eq(1) expect(stats[:recurring]).to eq(0) end + + it 'derives active_jobs from execution table counts' do + stats = described_class.calculate + + expected_active = stats[:ready] + stats[:scheduled] + stats[:in_progress] + stats[:failed] + expect(stats[:active_jobs]).to eq(expected_active) + end + + it 'does not query the jobs table for counts' do + expect(SolidQueue::Job).not_to receive(:count) + described_class.calculate + end end end From 61889489de2d8430f18b3fb01e2bf10a67b55337 Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Fri, 6 Mar 2026 21:25:37 +0530 Subject: [PATCH 2/7] perf: replace unbounded pluck with subqueries and fix N+1 queue stats - Change .pluck(:job_id) to .select(:job_id) in all filter methods to keep filtering as DB subqueries instead of loading IDs into memory - Pre-aggregate queue stats with 3 GROUP BY queries, eliminating per-queue COUNT queries in QueuesPresenter - Add spec to prevent unbounded pluck regression --- .../solid_queue_monitor/base_controller.rb | 38 ++++++------------- .../in_progress_jobs_controller.rb | 6 +-- .../solid_queue_monitor/queues_controller.rb | 28 +++++++++----- .../solid_queue_monitor/queues_presenter.rb | 29 ++++---------- .../no_unbounded_pluck_spec.rb | 35 +++++++++++++++++ 5 files changed, 75 insertions(+), 61 deletions(-) create mode 100644 spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb diff --git a/app/controllers/solid_queue_monitor/base_controller.rb b/app/controllers/solid_queue_monitor/base_controller.rb index 9275610..87acb8f 100644 --- a/app/controllers/solid_queue_monitor/base_controller.rb +++ b/app/controllers/solid_queue_monitor/base_controller.rb @@ -91,17 +91,13 @@ def filter_jobs(relation) when 'completed' relation = relation.where.not(finished_at: nil) when 'failed' - failed_job_ids = SolidQueue::FailedExecution.pluck(:job_id) - relation = relation.where(id: failed_job_ids) + relation = relation.where(id: SolidQueue::FailedExecution.select(:job_id)) when 'scheduled' - scheduled_job_ids = SolidQueue::ScheduledExecution.pluck(:job_id) - relation = relation.where(id: scheduled_job_ids) + relation = relation.where(id: SolidQueue::ScheduledExecution.select(:job_id)) when 'pending' - # Pending jobs are those that are not completed, failed, or scheduled - failed_job_ids = SolidQueue::FailedExecution.pluck(:job_id) - scheduled_job_ids = SolidQueue::ScheduledExecution.pluck(:job_id) relation = relation.where(finished_at: nil) - .where.not(id: failed_job_ids + scheduled_job_ids) + .where.not(id: SolidQueue::FailedExecution.select(:job_id)) + .where.not(id: SolidQueue::ScheduledExecution.select(:job_id)) end end @@ -117,16 +113,13 @@ def filter_ready_jobs(relation) return relation unless params[:class_name].present? || params[:queue_name].present? || params[:arguments].present? if params[:class_name].present? - job_ids = SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").select(:id)) end relation = relation.where('queue_name LIKE ?', "%#{params[:queue_name]}%") if params[:queue_name].present? - # Add arguments filtering if params[:arguments].present? - job_ids = SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").select(:id)) end relation @@ -136,16 +129,13 @@ def filter_scheduled_jobs(relation) return relation unless params[:class_name].present? || params[:queue_name].present? || params[:arguments].present? if params[:class_name].present? - job_ids = SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").select(:id)) end relation = relation.where('queue_name LIKE ?', "%#{params[:queue_name]}%") if params[:queue_name].present? - # Add arguments filtering if params[:arguments].present? - job_ids = SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").select(:id)) end relation @@ -170,25 +160,19 @@ def filter_failed_jobs(relation) return relation unless params[:class_name].present? || params[:queue_name].present? || params[:arguments].present? if params[:class_name].present? - job_ids = SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").select(:id)) end if params[:queue_name].present? - # Check if FailedExecution has queue_name column if relation.column_names.include?('queue_name') relation = relation.where('queue_name LIKE ?', "%#{params[:queue_name]}%") else - # If not, filter by job's queue_name - job_ids = SolidQueue::Job.where('queue_name LIKE ?', "%#{params[:queue_name]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('queue_name LIKE ?', "%#{params[:queue_name]}%").select(:id)) end end - # Add arguments filtering if params[:arguments].present? - job_ids = SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").select(:id)) end relation diff --git a/app/controllers/solid_queue_monitor/in_progress_jobs_controller.rb b/app/controllers/solid_queue_monitor/in_progress_jobs_controller.rb index 48e3bf5..b01c4c0 100644 --- a/app/controllers/solid_queue_monitor/in_progress_jobs_controller.rb +++ b/app/controllers/solid_queue_monitor/in_progress_jobs_controller.rb @@ -22,13 +22,11 @@ def filter_in_progress_jobs(relation) return relation if params[:class_name].blank? && params[:arguments].blank? if params[:class_name].present? - job_ids = SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").select(:id)) end if params[:arguments].present? - job_ids = SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").pluck(:id) - relation = relation.where(job_id: job_ids) + relation = relation.where(job_id: SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").select(:id)) end relation diff --git a/app/controllers/solid_queue_monitor/queues_controller.rb b/app/controllers/solid_queue_monitor/queues_controller.rb index 2a3764d..eed52ac 100644 --- a/app/controllers/solid_queue_monitor/queues_controller.rb +++ b/app/controllers/solid_queue_monitor/queues_controller.rb @@ -10,8 +10,13 @@ def index .select('queue_name, COUNT(*) as job_count') @queues = apply_queue_sorting(base_query) @paused_queues = QueuePauseService.paused_queues + @queue_stats = aggregate_queue_stats - render_page('Queues', SolidQueueMonitor::QueuesPresenter.new(@queues, @paused_queues, sort: sort_params).render) + render_page('Queues', SolidQueueMonitor::QueuesPresenter.new( + @queues, @paused_queues, + queue_stats: @queue_stats, + sort: sort_params + ).render) end def show @@ -57,6 +62,15 @@ def resume private + def aggregate_queue_stats + { + ready: SolidQueue::ReadyExecution.group(:queue_name).count, + scheduled: SolidQueue::ScheduledExecution.group(:queue_name).count, + failed: SolidQueue::FailedExecution.joins(:job) + .group('solid_queue_jobs.queue_name').count + } + end + def calculate_queue_counts(queue_name) { total: SolidQueue::Job.where(queue_name: queue_name).count, @@ -77,17 +91,13 @@ def filter_queue_jobs(relation) when 'completed' relation = relation.where.not(finished_at: nil) when 'failed' - failed_job_ids = SolidQueue::FailedExecution.pluck(:job_id) - relation = relation.where(id: failed_job_ids) + relation = relation.where(id: SolidQueue::FailedExecution.select(:job_id)) when 'scheduled' - scheduled_job_ids = SolidQueue::ScheduledExecution.pluck(:job_id) - relation = relation.where(id: scheduled_job_ids) + relation = relation.where(id: SolidQueue::ScheduledExecution.select(:job_id)) when 'pending' - ready_job_ids = SolidQueue::ReadyExecution.pluck(:job_id) - relation = relation.where(id: ready_job_ids) + relation = relation.where(id: SolidQueue::ReadyExecution.select(:job_id)) when 'in_progress' - claimed_job_ids = SolidQueue::ClaimedExecution.pluck(:job_id) - relation = relation.where(id: claimed_job_ids) + relation = relation.where(id: SolidQueue::ClaimedExecution.select(:job_id)) end end diff --git a/app/presenters/solid_queue_monitor/queues_presenter.rb b/app/presenters/solid_queue_monitor/queues_presenter.rb index ed5152a..2c60282 100644 --- a/app/presenters/solid_queue_monitor/queues_presenter.rb +++ b/app/presenters/solid_queue_monitor/queues_presenter.rb @@ -2,10 +2,11 @@ module SolidQueueMonitor class QueuesPresenter < BasePresenter - def initialize(records, paused_queues = [], sort: {}) - @records = records + def initialize(records, paused_queues = [], sort: {}, queue_stats: {}) + @records = records @paused_queues = paused_queues - @sort = sort + @sort = sort + @queue_stats = queue_stats end def render @@ -39,16 +40,16 @@ def generate_table def generate_row(queue) queue_name = queue.queue_name || 'default' - paused = @paused_queues.include?(queue_name) + paused = @paused_queues.include?(queue_name) <<-HTML #{queue_link(queue_name)} #{status_badge(paused)} #{queue.job_count} - #{ready_jobs_count(queue_name)} - #{scheduled_jobs_count(queue_name)} - #{failed_jobs_count(queue_name)} + #{@queue_stats.dig(:ready, queue_name) || 0} + #{@queue_stats.dig(:scheduled, queue_name) || 0} + #{@queue_stats.dig(:failed, queue_name) || 0} #{action_button(queue_name, paused)} HTML @@ -84,19 +85,5 @@ def action_button(queue_name, paused) HTML end end - - def ready_jobs_count(queue_name) - SolidQueue::ReadyExecution.where(queue_name: queue_name).count - end - - def scheduled_jobs_count(queue_name) - SolidQueue::ScheduledExecution.where(queue_name: queue_name).count - end - - def failed_jobs_count(queue_name) - SolidQueue::FailedExecution.joins(:job) - .where(solid_queue_jobs: { queue_name: queue_name }) - .count - end end end diff --git a/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb b/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb new file mode 100644 index 0000000..697fd88 --- /dev/null +++ b/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'No unbounded pluck calls in controllers' do + root = File.expand_path('../../..', __dir__) + controller_files = Dir[File.join(root, 'app', 'controllers', '**', '*.rb')] + + raise "No controller files found from #{root}" if controller_files.empty? + + controller_files.each do |file| + relative = file.sub("#{root}/", '') + + it "#{relative} does not use unbounded pluck for subquery filters" do + content = File.read(file) + + # Catch patterns like: + # SolidQueue::FailedExecution.pluck(:job_id) + # SolidQueue::Job.where(...).pluck(:id) + # But NOT bounded plucks like: + # SolidQueue::FailedExecution.where(job_id: job_ids).pluck(:job_id) + # The distinction: unbounded plucks are used to build WHERE IN arrays for filtering. + # We detect lines that assign pluck results to variables used in .where(id: ...) patterns. + pluck_filter_lines = content.lines.select do |line| + line.match?(/=\s*SolidQueue::\w+(\.\w+\(.*\))*\.pluck\(:(?:job_)?id\)/) && + !line.match?(/\.where\(job_id:/) + end + + expect(pluck_filter_lines).to be_empty, + "Found unbounded pluck calls used for filtering in #{relative}:\n" \ + "#{pluck_filter_lines.map(&:strip).join("\n")}\n" \ + "Use .select(:job_id) or .select(:id) for subqueries instead." + end + end +end From 8bf898b04b842037b7ca9d1199e34c21e0896e9d Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Fri, 6 Mar 2026 21:25:56 +0530 Subject: [PATCH 3/7] perf: use SQL GROUP BY bucketing for chart data Replace in-memory timestamp bucketing (pluck all timestamps, iterate in Ruby) with SQL GROUP BY using computed bucket index. Works on both PostgreSQL and SQLite with adapter-aware expressions. --- .../solid_queue_monitor/chart_data_service.rb | 112 ++++++------ .../chart_data_service_spec.rb | 166 +++++++----------- 2 files changed, 117 insertions(+), 161 deletions(-) diff --git a/app/services/solid_queue_monitor/chart_data_service.rb b/app/services/solid_queue_monitor/chart_data_service.rb index 0315a49..b1035ef 100644 --- a/app/services/solid_queue_monitor/chart_data_service.rb +++ b/app/services/solid_queue_monitor/chart_data_service.rb @@ -5,48 +5,43 @@ class ChartDataService TIME_RANGES = { '15m' => { duration: 15.minutes, buckets: 15, label_format: '%H:%M', label: 'Last 15 minutes' }, '30m' => { duration: 30.minutes, buckets: 15, label_format: '%H:%M', label: 'Last 30 minutes' }, - '1h' => { duration: 1.hour, buckets: 12, label_format: '%H:%M', label: 'Last 1 hour' }, - '3h' => { duration: 3.hours, buckets: 18, label_format: '%H:%M', label: 'Last 3 hours' }, - '6h' => { duration: 6.hours, buckets: 24, label_format: '%H:%M', label: 'Last 6 hours' }, - '12h' => { duration: 12.hours, buckets: 24, label_format: '%H:%M', label: 'Last 12 hours' }, - '1d' => { duration: 1.day, buckets: 24, label_format: '%H:%M', label: 'Last 24 hours' }, - '3d' => { duration: 3.days, buckets: 36, label_format: '%m/%d %H:%M', label: 'Last 3 days' }, - '1w' => { duration: 7.days, buckets: 28, label_format: '%m/%d', label: 'Last 7 days' } + '1h' => { duration: 1.hour, buckets: 12, label_format: '%H:%M', label: 'Last 1 hour' }, + '3h' => { duration: 3.hours, buckets: 18, label_format: '%H:%M', label: 'Last 3 hours' }, + '6h' => { duration: 6.hours, buckets: 24, label_format: '%H:%M', label: 'Last 6 hours' }, + '12h' => { duration: 12.hours, buckets: 24, label_format: '%H:%M', label: 'Last 12 hours' }, + '1d' => { duration: 1.day, buckets: 24, label_format: '%H:%M', label: 'Last 24 hours' }, + '3d' => { duration: 3.days, buckets: 36, label_format: '%m/%d %H:%M', label: 'Last 3 days' }, + '1w' => { duration: 7.days, buckets: 28, label_format: '%m/%d', label: 'Last 7 days' } }.freeze DEFAULT_TIME_RANGE = '1d' def initialize(time_range: DEFAULT_TIME_RANGE) @time_range = TIME_RANGES.key?(time_range) ? time_range : DEFAULT_TIME_RANGE - @config = TIME_RANGES[@time_range] + @config = TIME_RANGES[@time_range] end def calculate - end_time = Time.current - start_time = end_time - @config[:duration] - bucket_duration = @config[:duration] / @config[:buckets] + end_time = Time.current + start_time = end_time - @config[:duration] + bucket_seconds = (@config[:duration] / @config[:buckets]).to_i + buckets = build_buckets(start_time, bucket_seconds) - buckets = build_buckets(start_time, bucket_duration) + created_data = bucket_counts(SolidQueue::Job, :created_at, start_time, end_time, bucket_seconds) + completed_data = bucket_counts(SolidQueue::Job, :finished_at, start_time, end_time, bucket_seconds, exclude_nil: true) + failed_data = bucket_counts(SolidQueue::FailedExecution, :created_at, start_time, end_time, bucket_seconds) - created_counts = fetch_created_counts(start_time, end_time) - completed_counts = fetch_completed_counts(start_time, end_time) - failed_counts = fetch_failed_counts(start_time, end_time) - - created_data = assign_to_buckets(created_counts, buckets, bucket_duration) - completed_data = assign_to_buckets(completed_counts, buckets, bucket_duration) - failed_data = assign_to_buckets(failed_counts, buckets, bucket_duration) + created_arr = fill_buckets(buckets, created_data) + completed_arr = fill_buckets(buckets, completed_data) + failed_arr = fill_buckets(buckets, failed_data) { - labels: buckets.map { |b| b[:label] }, # rubocop:disable Rails/Pluck - created: created_data, - completed: completed_data, - failed: failed_data, - totals: { - created: created_data.sum, - completed: completed_data.sum, - failed: failed_data.sum - }, - time_range: @time_range, + labels: buckets.map { |b| b[:label] }, # rubocop:disable Rails/Pluck + created: created_arr, + completed: completed_arr, + failed: failed_arr, + totals: { created: created_arr.sum, completed: completed_arr.sum, failed: failed_arr.sum }, + time_range: @time_range, time_range_label: @config[:label], available_ranges: TIME_RANGES.transform_values { |v| v[:label] } } @@ -54,47 +49,46 @@ def calculate private - def build_buckets(start_time, bucket_duration) + def build_buckets(start_time, bucket_seconds) @config[:buckets].times.map do |i| - bucket_start = start_time + (i * bucket_duration) - { - start: bucket_start, - end: bucket_start + bucket_duration, - label: bucket_start.strftime(@config[:label_format]) - } + bucket_start = start_time + (i * bucket_seconds) + { index: i, start: bucket_start, label: bucket_start.strftime(@config[:label_format]) } end end - def fetch_created_counts(start_time, end_time) - SolidQueue::Job - .where(created_at: start_time..end_time) - .pluck(:created_at) - end + # Returns a Hash of { bucket_index => count } using SQL GROUP BY. + # The bucket index is computed as: (epoch(column) - epoch(start_time)) / interval + # This works identically on PostgreSQL and SQLite. + def bucket_counts(model, column, start_time, end_time, interval, exclude_nil: false) + start_epoch = start_time.to_i + expr = bucket_index_expr(column, start_epoch, interval) - def fetch_completed_counts(start_time, end_time) - SolidQueue::Job - .where(finished_at: start_time..end_time) - .where.not(finished_at: nil) - .pluck(:finished_at) - end + scope = model.where(column => start_time..end_time) + scope = scope.where.not(column => nil) if exclude_nil - def fetch_failed_counts(start_time, end_time) - SolidQueue::FailedExecution - .where(created_at: start_time..end_time) - .pluck(:created_at) + scope + .group(Arel.sql(expr)) + .pluck(Arel.sql("#{expr} AS bucket_idx, COUNT(*) AS cnt")) + .to_h { |idx, cnt| [idx.to_i, cnt] } end - def assign_to_buckets(timestamps, buckets, _bucket_duration) - counts = Array.new(buckets.size, 0) + def fill_buckets(buckets, index_counts) + buckets.map { |b| index_counts.fetch(b[:index], 0) } + end - timestamps.each do |timestamp| - bucket_index = buckets.find_index do |bucket| - timestamp >= bucket[:start] && timestamp < bucket[:end] - end - counts[bucket_index] += 1 if bucket_index + # Cross-DB bucket index expression. + # PostgreSQL: CAST((EXTRACT(EPOCH FROM col) - start) / interval AS INTEGER) + # SQLite: CAST((CAST(strftime('%s', col) AS INTEGER) - start) / interval AS INTEGER) + def bucket_index_expr(column, start_epoch, interval_seconds) + if sqlite? + "CAST((CAST(strftime('%s', #{column}) AS INTEGER) - #{start_epoch}) / #{interval_seconds} AS INTEGER)" + else + "CAST((EXTRACT(EPOCH FROM #{column}) - #{start_epoch}) / #{interval_seconds} AS INTEGER)" end + end - counts + def sqlite? + ActiveRecord::Base.connection.adapter_name.downcase.include?('sqlite') end end end diff --git a/spec/services/solid_queue_monitor/chart_data_service_spec.rb b/spec/services/solid_queue_monitor/chart_data_service_spec.rb index 03668ca..f831e6f 100644 --- a/spec/services/solid_queue_monitor/chart_data_service_spec.rb +++ b/spec/services/solid_queue_monitor/chart_data_service_spec.rb @@ -2,154 +2,116 @@ require 'spec_helper' -# rubocop:disable RSpec/VerifiedDoubles RSpec.describe SolidQueueMonitor::ChartDataService do describe '#calculate' do let(:service) { described_class.new(time_range: time_range) } let(:time_range) { '1d' } - before do - # Mock the created_at query chain - created_relation = double('created_relation') - allow(created_relation).to receive(:pluck).with(:created_at).and_return([]) - allow(SolidQueue::Job).to receive(:where).with(created_at: anything).and_return(created_relation) - - # Mock the finished_at query chain (where.where.not.pluck) - completed_relation = double('completed_relation') - completed_not_relation = double('completed_not_relation') - allow(completed_relation).to receive(:where).and_return(completed_not_relation) - allow(completed_not_relation).to receive(:not).and_return(completed_not_relation) - allow(completed_not_relation).to receive(:pluck).with(:finished_at).and_return([]) - allow(SolidQueue::Job).to receive(:where).with(finished_at: anything).and_return(completed_relation) - - # Mock the failed executions query - failed_relation = double('failed_relation') - allow(failed_relation).to receive(:pluck).with(:created_at).and_return([]) - allow(SolidQueue::FailedExecution).to receive(:where).with(created_at: anything).and_return(failed_relation) - end - - it 'returns chart data structure' do - result = service.calculate - - expect(result).to include( - :labels, - :created, - :completed, - :failed, - :totals, - :time_range, - :time_range_label, - :available_ranges - ) - end - - it 'returns correct number of buckets for 1d range' do - result = service.calculate - - expect(result[:labels].size).to eq(24) - expect(result[:created].size).to eq(24) - expect(result[:completed].size).to eq(24) - expect(result[:failed].size).to eq(24) - end + context 'with no data' do + it 'returns the required keys' do + result = service.calculate + expect(result).to include(:labels, :created, :completed, :failed, + :totals, :time_range, :time_range_label, :available_ranges) + end - it 'returns the current time range' do - result = service.calculate + it 'returns correct bucket count for 1d' do + result = service.calculate + expect(result[:labels].size).to eq(24) + expect(result[:created].size).to eq(24) + expect(result[:completed].size).to eq(24) + expect(result[:failed].size).to eq(24) + end - expect(result[:time_range]).to eq('1d') - end + it 'returns all zeros' do + result = service.calculate + expect(result[:totals]).to eq({ created: 0, completed: 0, failed: 0 }) + end - it 'returns all available time ranges with labels' do - result = service.calculate + it 'returns the current time range' do + expect(service.calculate[:time_range]).to eq('1d') + end - expect(result[:available_ranges].keys).to eq(%w[15m 30m 1h 3h 6h 12h 1d 3d 1w]) + it 'returns all available time ranges' do + expect(service.calculate[:available_ranges].keys).to eq(%w[15m 30m 1h 3h 6h 12h 1d 3d 1w]) + end end context 'with 1h time range' do let(:time_range) { '1h' } - - it 'returns 12 buckets' do - result = service.calculate - - expect(result[:labels].size).to eq(12) - end + it('returns 12 buckets') { expect(service.calculate[:labels].size).to eq(12) } end context 'with 1w time range' do let(:time_range) { '1w' } - - it 'returns 28 buckets' do - result = service.calculate - - expect(result[:labels].size).to eq(28) - end + it('returns 28 buckets') { expect(service.calculate[:labels].size).to eq(28) } end context 'with invalid time range' do let(:time_range) { 'invalid' } - it 'defaults to 1d' do + it 'defaults to 1d with 24 buckets' do result = service.calculate - expect(result[:time_range]).to eq('1d') expect(result[:labels].size).to eq(24) end end - context 'with job data' do - let(:now) { Time.current } - let(:created_timestamps) { [now - 30.minutes, now - 45.minutes] } - let(:completed_timestamps) { [now - 20.minutes] } - let(:failed_timestamps) { [now - 10.minutes, now - 15.minutes] } + context 'with jobs in the time window' do + let(:time_range) { '1h' } before do - # Override mocks with actual data - created_relation = double('created_relation') - allow(created_relation).to receive(:pluck).with(:created_at).and_return(created_timestamps) - allow(SolidQueue::Job).to receive(:where).with(created_at: anything).and_return(created_relation) - - completed_relation = double('completed_relation') - completed_not_relation = double('completed_not_relation') - allow(completed_relation).to receive(:where).and_return(completed_not_relation) - allow(completed_not_relation).to receive(:not).and_return(completed_not_relation) - allow(completed_not_relation).to receive(:pluck).with(:finished_at).and_return(completed_timestamps) - allow(SolidQueue::Job).to receive(:where).with(finished_at: anything).and_return(completed_relation) - - failed_relation = double('failed_relation') - allow(failed_relation).to receive(:pluck).with(:created_at).and_return(failed_timestamps) - allow(SolidQueue::FailedExecution).to receive(:where).with(created_at: anything).and_return(failed_relation) + now = Time.current + create(:solid_queue_job, created_at: now - 10.minutes) + create(:solid_queue_job, created_at: now - 10.minutes) + create(:solid_queue_job, :completed, + created_at: now - 25.minutes, finished_at: now - 20.minutes) + create(:solid_queue_failed_execution, created_at: now - 15.minutes) + end + + it 'counts created jobs' do + # At least 2 regular + 1 completed + 1 from failed execution factory + expect(service.calculate[:created].sum).to be >= 2 + end + + it 'counts completed jobs' do + expect(service.calculate[:completed].sum).to eq(1) + end + + it 'counts failed executions' do + expect(service.calculate[:failed].sum).to eq(1) end - it 'aggregates job counts into buckets' do + it 'totals match bucket sums' do result = service.calculate + expect(result[:totals][:created]).to eq(result[:created].sum) + expect(result[:totals][:completed]).to eq(result[:completed].sum) + expect(result[:totals][:failed]).to eq(result[:failed].sum) + end + end - total_created = result[:created].sum - total_completed = result[:completed].sum - total_failed = result[:failed].sum + context 'with jobs outside the window' do + let(:time_range) { '1h' } + before { create(:solid_queue_job, created_at: 2.hours.ago) } - expect(total_created).to eq(2) - expect(total_completed).to eq(1) - expect(total_failed).to eq(2) + it 'excludes them' do + expect(service.calculate[:created].sum).to eq(0) end end end - describe 'TIME_RANGES' do - it 'defines all expected time ranges' do + describe 'constants' do + it 'defines all time ranges' do expect(described_class::TIME_RANGES.keys).to eq(%w[15m 30m 1h 3h 6h 12h 1d 3d 1w]) end - it 'has duration, buckets, label_format, and label for each range' do - described_class::TIME_RANGES.each do |key, config| - expect(config).to include(:duration, :buckets, :label_format, :label), - "Expected #{key} to have duration, buckets, label_format, and label" + it 'has required config per range' do + described_class::TIME_RANGES.each_value do |config| + expect(config).to include(:duration, :buckets, :label_format, :label) end end - end - describe 'DEFAULT_TIME_RANGE' do - it 'is 1d' do + it 'defaults to 1d' do expect(described_class::DEFAULT_TIME_RANGE).to eq('1d') end end end -# rubocop:enable RSpec/VerifiedDoubles From 64adf2883e20dedba7c88845908856da4d0446a5 Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Fri, 6 Mar 2026 21:26:04 +0530 Subject: [PATCH 4/7] feat: add config.show_chart toggle to disable chart queries When show_chart is false, ChartDataService is not instantiated and the chart section is not rendered, eliminating chart queries entirely for users who don't need the visualization. --- .../overview_controller.rb | 16 +++++++------- .../templates/initializer.rb | 3 +++ lib/solid_queue_monitor.rb | 3 ++- .../solid_queue_monitor/overview_spec.rb | 22 ++++++++++++++++++- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/app/controllers/solid_queue_monitor/overview_controller.rb b/app/controllers/solid_queue_monitor/overview_controller.rb index 3ade073..2307134 100644 --- a/app/controllers/solid_queue_monitor/overview_controller.rb +++ b/app/controllers/solid_queue_monitor/overview_controller.rb @@ -6,7 +6,7 @@ class OverviewController < BaseController def index @stats = SolidQueueMonitor::StatsCalculator.calculate - @chart_data = SolidQueueMonitor::ChartDataService.new(time_range: time_range_param).calculate + @chart_data = SolidQueueMonitor.show_chart ? SolidQueueMonitor::ChartDataService.new(time_range: time_range_param).calculate : nil recent_jobs_query = SolidQueue::Job.limit(100) sorted_query = apply_sorting(filter_jobs(recent_jobs_query), SORTABLE_COLUMNS, 'created_at', :desc) @@ -29,13 +29,13 @@ def time_range_param end def generate_overview_content - SolidQueueMonitor::StatsPresenter.new(@stats).render + - SolidQueueMonitor::ChartPresenter.new(@chart_data).render + - SolidQueueMonitor::JobsPresenter.new(@recent_jobs[:records], - current_page: @recent_jobs[:current_page], - total_pages: @recent_jobs[:total_pages], - filters: filter_params, - sort: sort_params).render + html = SolidQueueMonitor::StatsPresenter.new(@stats).render + html += SolidQueueMonitor::ChartPresenter.new(@chart_data).render if @chart_data + html + SolidQueueMonitor::JobsPresenter.new(@recent_jobs[:records], + current_page: @recent_jobs[:current_page], + total_pages: @recent_jobs[:total_pages], + filters: filter_params, + sort: sort_params).render end end end diff --git a/lib/generators/solid_queue_monitor/templates/initializer.rb b/lib/generators/solid_queue_monitor/templates/initializer.rb index a3af664..3e56296 100644 --- a/lib/generators/solid_queue_monitor/templates/initializer.rb +++ b/lib/generators/solid_queue_monitor/templates/initializer.rb @@ -20,4 +20,7 @@ # Auto-refresh interval in seconds (default: 30) # config.auto_refresh_interval = 30 + + # Disable the chart on the overview page to skip chart queries entirely. + # config.show_chart = true end diff --git a/lib/solid_queue_monitor.rb b/lib/solid_queue_monitor.rb index 2ee8df9..ddc7b83 100644 --- a/lib/solid_queue_monitor.rb +++ b/lib/solid_queue_monitor.rb @@ -7,7 +7,7 @@ module SolidQueueMonitor class Error < StandardError; end class << self attr_accessor :username, :password, :jobs_per_page, :authentication_enabled, - :auto_refresh_enabled, :auto_refresh_interval + :auto_refresh_enabled, :auto_refresh_interval, :show_chart end @username = 'admin' @@ -16,6 +16,7 @@ class << self @authentication_enabled = false @auto_refresh_enabled = true @auto_refresh_interval = 30 # seconds + @show_chart = true def self.setup yield self diff --git a/spec/requests/solid_queue_monitor/overview_spec.rb b/spec/requests/solid_queue_monitor/overview_spec.rb index d5d84b1..807bfa2 100644 --- a/spec/requests/solid_queue_monitor/overview_spec.rb +++ b/spec/requests/solid_queue_monitor/overview_spec.rb @@ -29,7 +29,7 @@ get '/' expect(response.body).to include('Queue Statistics') - expect(response.body).to include('Total Jobs') + expect(response.body).to include('Active Jobs') end it 'displays navigation links' do @@ -41,6 +41,26 @@ end end + context 'with chart disabled' do + around do |example| + original = SolidQueueMonitor.show_chart + SolidQueueMonitor.show_chart = false + example.run + SolidQueueMonitor.show_chart = original + end + + it 'does not call ChartDataService' do + expect(SolidQueueMonitor::ChartDataService).not_to receive(:new) + get '/' + expect(response).to have_http_status(:ok) + end + + it 'does not render chart section' do + get '/' + expect(response.body).not_to include('id="chart-section"') + end + end + context 'with authentication enabled' do before do allow(SolidQueueMonitor).to receive_messages(authentication_enabled: true, username: 'admin', password: 'password123') From c5fb60e948bce96a55427716f6e84f3af3c8b5ad Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Fri, 6 Mar 2026 21:26:21 +0530 Subject: [PATCH 5/7] docs: document performance optimizations and show_chart config - Add "Performance at Scale" section to README - Add [Unreleased] section to CHANGELOG with breaking change note - Add Large Dataset Performance to ROADMAP as done - Include implementation plan in docs/plans/ --- CHANGELOG.md | 17 + README.md | 18 + ROADMAP.md | 1 + .../2026-03-06-performance-large-datasets.md | 1033 +++++++++++++++++ 4 files changed, 1069 insertions(+) create mode 100644 docs/plans/2026-03-06-performance-large-datasets.md diff --git a/CHANGELOG.md b/CHANGELOG.md index aa59930..e346e8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## [Unreleased] + +### Changed + +- **BREAKING**: Dashboard "Total Jobs" and "Completed" stats replaced with "Active Jobs" (sum of ready + scheduled + in-progress + failed). This avoids expensive `COUNT(*)` on the jobs table at scale. + +### Fixed + +- **Performance**: Overview page no longer queries `solid_queue_jobs` for stats — all counts derived from execution tables (resolves gateway timeouts with millions of rows) ([#27](https://github.com/vishaltps/solid_queue_monitor/issues/27)) +- **Performance**: Chart data service uses SQL `GROUP BY` bucketing instead of loading all timestamps into Ruby memory +- **Performance**: All filter methods use `.select(:job_id)` subqueries instead of unbounded `.pluck(:job_id)` +- **Performance**: Queue stats pre-aggregated with 3 `GROUP BY` queries, eliminating N+1 per-queue COUNT queries + +### Added + +- `config.show_chart` option to disable the job activity chart and skip chart queries entirely + ## [1.1.0] - 2026-02-07 ### Added diff --git a/README.md b/README.md index 7614523..4d448da 100644 --- a/README.md +++ b/README.md @@ -118,9 +118,27 @@ SolidQueueMonitor.setup do |config| # Auto-refresh interval in seconds (default: 30) config.auto_refresh_interval = 30 + + # Disable the chart on the overview page to skip chart queries entirely + # config.show_chart = true end ``` +### Performance at Scale + +SolidQueueMonitor is optimized for large datasets (millions of rows in `solid_queue_jobs`): + +- **Overview stats** are derived entirely from execution tables (`ready_executions`, `scheduled_executions`, `claimed_executions`, `failed_executions`), avoiding expensive `COUNT(*)` queries on the jobs table. +- **Chart data** uses SQL `GROUP BY` bucketing instead of loading timestamps into Ruby memory. +- **Filters** use subqueries (`.select(:job_id)`) instead of loading ID arrays into memory. +- **Queue stats** are pre-aggregated with `GROUP BY` to avoid N+1 queries. + +If you don't need the job activity chart, disable it to skip chart queries entirely: + +```ruby +config.show_chart = false +``` + ### Authentication By default, Solid Queue Monitor does not require authentication to access the dashboard. This makes it easy to get started in development environments. diff --git a/ROADMAP.md b/ROADMAP.md index 9f06dbc..5652af4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -22,6 +22,7 @@ This document tracks planned features for solid_queue_monitor, comparing with ot | Job Details Page | Dedicated page for single job with full context | ✅ Done | | Search/Full-text Search | Better search across all job data | ✅ Done | | Sorting Options | Sort by various columns | ✅ Done | +| Large Dataset Performance | Optimized for millions of rows — no jobs table scans | ✅ Done | | Backtrace Cleaner | Remove framework noise from error backtraces | ⬚ Planned | | Manual Job Triggering | Enqueue a job directly from the dashboard | ⬚ Planned | | Cancel Running Jobs | Stop long-running jobs | ⬚ Planned | diff --git a/docs/plans/2026-03-06-performance-large-datasets.md b/docs/plans/2026-03-06-performance-large-datasets.md new file mode 100644 index 0000000..e54971a --- /dev/null +++ b/docs/plans/2026-03-06-performance-large-datasets.md @@ -0,0 +1,1033 @@ +# Performance: Large Dataset Support Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Fix gateway timeouts on large Solid Queue datasets (4M+ rows in `solid_queue_jobs`). Make the default experience fast without any configuration. + +**Architecture:** Five tasks, ordered by impact. Tasks 1–4 are unconditional fixes — they make things faster for everyone without configuration flags or behaviour changes. Task 5 adds a single opt-in config (`show_chart`) for teams that want to eliminate chart queries entirely. No `approximate_counts`, no `paginate_without_count`, no `root_redirect_to` — those are complexity for problems we can solve properly. + +**Tech Stack:** Ruby/Rails engine, RSpec, FactoryBot. Tests run on **SQLite in-memory** (`spec_helper.rb:16-18`). Production users are on PostgreSQL. All SQL must work on both. + +**Ref:** GitHub issue #27 + +--- + +## Why the Sonnet Plan Was Wrong + +| Sonnet Proposal | Problem | +|-----------------|---------| +| `pg_class.reltuples` for approximate counts | PostgreSQL-only. Tests run on SQLite. Adds a config flag to work around a problem we can eliminate. | +| `EXTRACT(EPOCH FROM ...)` in ChartDataService | PostgreSQL-only. Breaks test suite. | +| `paginate_without_count` config | Pagination COUNT is only expensive on `solid_queue_jobs`. If we stop paginating that table at scale, the problem disappears. | +| `approximate_counts` config | If we stop running COUNT(*) on `solid_queue_jobs` at all, there's nothing to approximate. | +| Fixed plucks in `filter_jobs` only | Missed 8+ identical pluck calls in `filter_ready_jobs`, `filter_scheduled_jobs`, `filter_failed_jobs`, `filter_in_progress_jobs`. | + +## Root Cause Analysis + +The real question: **why does the overview page scan `solid_queue_jobs` at all?** + +`StatsCalculator` runs three queries against `solid_queue_jobs`: +1. `SolidQueue::Job.count` → `total_jobs` (52 seconds at 4M rows) +2. `SolidQueue::Job.distinct.count(:queue_name)` → `unique_queues` +3. `SolidQueue::Job.where.not(finished_at: nil).count` → `completed` + +But what does an operator actually need? **How many jobs are ready, failed, in-progress, scheduled.** Those all come from execution tables which are small. `total_jobs` (including millions of finished jobs) is vanity data — not operationally actionable. + +**Solution: Stop querying the jobs table for stats.** Derive everything from execution tables. This eliminates the 52-second queries entirely — no config flag, no PostgreSQL-specific hacks, just better code. + +--- + +## Task 1: Rewrite StatsCalculator to avoid jobs table COUNT + +**Why this is the highest-impact fix.** Three queries on `solid_queue_jobs` account for ~156 seconds of the timeout. Removing them solves the core complaint. + +**Files:** +- Modify: `app/services/solid_queue_monitor/stats_calculator.rb` +- Modify: `app/presenters/solid_queue_monitor/stats_presenter.rb` +- Modify: `spec/services/solid_queue_monitor/stats_calculator_spec.rb` +- Modify: `spec/presenters/solid_queue_monitor/stats_presenter_spec.rb` +- Modify: `spec/requests/solid_queue_monitor/overview_spec.rb` + +**Design decision:** Replace `total_jobs` (COUNT on jobs table) and `completed` (COUNT with WHERE on jobs table) with stats derived entirely from execution tables: +- Remove `total_jobs` — it's meaningless at 4M rows (it includes all historical finished jobs). +- Replace with `active_jobs` = `ready + scheduled + in_progress + failed` — what operators actually care about. +- Remove `completed` — requires scanning jobs table. At 4M scale, most rows are completed. Not useful. +- Remove `unique_queues` — `SolidQueue::Job.distinct.count(:queue_name)` scans the full table. + +The new stat cards: **Active Jobs**, **Ready**, **In Progress**, **Scheduled**, **Recurring**, **Failed**. + +Every single query hits small execution tables. Zero queries on `solid_queue_jobs`. + +--- + +**Step 1: Update the spec** + +Replace `spec/services/solid_queue_monitor/stats_calculator_spec.rb`: + +```ruby +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe SolidQueueMonitor::StatsCalculator do + describe '.calculate' do + before do + create(:solid_queue_failed_execution) + create(:solid_queue_scheduled_execution) + create(:solid_queue_ready_execution) + create(:solid_queue_claimed_execution) + end + + it 'returns a hash with all required statistics' do + stats = described_class.calculate + + expect(stats).to include( + :active_jobs, + :scheduled, + :ready, + :failed, + :in_progress, + :recurring + ) + end + + it 'calculates the correct counts from execution tables' do + stats = described_class.calculate + + expect(stats[:scheduled]).to eq(1) + expect(stats[:ready]).to eq(1) + expect(stats[:failed]).to eq(1) + expect(stats[:in_progress]).to eq(1) + expect(stats[:recurring]).to eq(0) + end + + it 'derives active_jobs from execution table counts' do + stats = described_class.calculate + + expected_active = stats[:ready] + stats[:scheduled] + stats[:in_progress] + stats[:failed] + expect(stats[:active_jobs]).to eq(expected_active) + end + + it 'does not query the jobs table for counts' do + expect(SolidQueue::Job).not_to receive(:count) + described_class.calculate + end + end +end +``` + +**Step 2: Run to confirm failure** + +```bash +bundle exec rspec spec/services/solid_queue_monitor/stats_calculator_spec.rb -f doc +``` + +Expected: FAIL — old calculator still returns `:total_jobs`, `:completed`, `:unique_queues` and queries `SolidQueue::Job`. + +**Step 3: Rewrite StatsCalculator** + +Replace `app/services/solid_queue_monitor/stats_calculator.rb`: + +```ruby +# frozen_string_literal: true + +module SolidQueueMonitor + class StatsCalculator + def self.calculate + scheduled = SolidQueue::ScheduledExecution.count + ready = SolidQueue::ReadyExecution.count + failed = SolidQueue::FailedExecution.count + in_progress = SolidQueue::ClaimedExecution.count + recurring = SolidQueue::RecurringTask.count + + { + active_jobs: ready + scheduled + in_progress + failed, + scheduled: scheduled, + ready: ready, + failed: failed, + in_progress: in_progress, + recurring: recurring + } + end + end +end +``` + +**Step 4: Update StatsPresenter** + +Replace `app/presenters/solid_queue_monitor/stats_presenter.rb`: + +```ruby +# frozen_string_literal: true + +module SolidQueueMonitor + class StatsPresenter < BasePresenter + def initialize(stats) + @stats = stats + end + + def render + <<-HTML +
+

Queue Statistics

+
+ #{generate_stat_card('Active Jobs', @stats[:active_jobs])} + #{generate_stat_card('Ready', @stats[:ready])} + #{generate_stat_card('In Progress', @stats[:in_progress])} + #{generate_stat_card('Scheduled', @stats[:scheduled])} + #{generate_stat_card('Recurring', @stats[:recurring])} + #{generate_stat_card('Failed', @stats[:failed])} +
+
+ HTML + end + + private + + def generate_stat_card(title, value) + <<-HTML +
+

#{title}

+

#{value}

+
+ HTML + end + end +end +``` + +**Step 5: Update the overview request spec and stats_presenter_spec if they assert `Total Jobs` or `Completed`** + +In `spec/requests/solid_queue_monitor/overview_spec.rb`, change: +```ruby +expect(response.body).to include('Total Jobs') +``` +to: +```ruby +expect(response.body).to include('Active Jobs') +``` + +In `spec/presenters/solid_queue_monitor/stats_presenter_spec.rb`, update expectations to match the new stat keys. + +**Step 6: Run tests** + +```bash +bundle exec rspec spec/services/solid_queue_monitor/stats_calculator_spec.rb \ + spec/presenters/solid_queue_monitor/stats_presenter_spec.rb \ + spec/requests/solid_queue_monitor/overview_spec.rb -f doc +``` + +Expected: All PASS + +**Step 7: Run full suite** + +```bash +bundle exec rspec +``` + +Expected: All passing + +**Step 8: Commit** + +```bash +git add app/services/solid_queue_monitor/stats_calculator.rb \ + app/presenters/solid_queue_monitor/stats_presenter.rb \ + spec/services/solid_queue_monitor/stats_calculator_spec.rb \ + spec/presenters/solid_queue_monitor/stats_presenter_spec.rb \ + spec/requests/solid_queue_monitor/overview_spec.rb +git commit -m "perf: rewrite StatsCalculator to avoid jobs table entirely + +The overview page was running 3 COUNT queries on solid_queue_jobs +(total_jobs, completed, unique_queues), each taking ~52s at 4M rows. + +Replaced with execution-table-only stats: active_jobs (derived sum), +ready, in_progress, scheduled, failed, recurring. All queries now hit +small execution tables — microseconds, not minutes. + +Resolves the primary cause of gateway timeouts in issue #27." +``` + +--- + +## Task 2: Fix all unbounded pluck calls across all controllers + +**Why:** Every `pluck(:job_id)` / `pluck(:id)` loads the entire result into a Ruby Array, then generates a massive `WHERE IN (...)` clause. Using `select(:job_id)` keeps it as a subquery executed entirely in the DB. + +**Scope:** The Sonnet plan only fixed `filter_jobs` and `filter_queue_jobs`. There are **10+** identical pluck calls across `filter_ready_jobs`, `filter_scheduled_jobs`, `filter_failed_jobs`, and `filter_in_progress_jobs`. + +**Files:** +- Modify: `app/controllers/solid_queue_monitor/base_controller.rb` +- Modify: `app/controllers/solid_queue_monitor/queues_controller.rb` +- Modify: `app/controllers/solid_queue_monitor/in_progress_jobs_controller.rb` + +--- + +**Step 1: Write a test that catches pluck calls** + +Create `spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb`: + +```ruby +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'No unbounded pluck calls in controllers' do + # This test greps the source code to ensure we never use .pluck(:job_id) or .pluck(:id) + # in filter methods, which would load all IDs into memory. + controller_files = Dir[File.expand_path('../../../../app/controllers/**/*.rb', __dir__)] + + controller_files.each do |file| + relative = file.sub(%r{.*/app/}, 'app/') + + it "#{relative} does not use unbounded pluck in filter methods" do + content = File.read(file) + + # Match pluck calls that are NOT scoped by a bounded set (like where(job_id: job_ids)) + # We want to catch: SolidQueue::Something.pluck(:job_id) + # and: SolidQueue::Job.where(...).pluck(:id) + pluck_calls = content.scan(/\.pluck\(:(?:job_)?id\)/) + + expect(pluck_calls).to be_empty, + "Found unbounded pluck calls in #{relative}: #{pluck_calls.inspect}. " \ + "Use .select(:job_id) or .select(:id) for subqueries instead." + end + end +end +``` + +**Step 2: Run to see all failures** + +```bash +bundle exec rspec spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb -f doc +``` + +Expected: Multiple failures across base_controller.rb, queues_controller.rb, in_progress_jobs_controller.rb + +**Step 3: Fix `base_controller.rb`** + +Replace every `pluck(:job_id)` / `pluck(:id)` with `select(:job_id)` / `select(:id)`: + +In `filter_jobs` (lines 84–109): +```ruby +when 'failed' + relation = relation.where(id: SolidQueue::FailedExecution.select(:job_id)) +when 'scheduled' + relation = relation.where(id: SolidQueue::ScheduledExecution.select(:job_id)) +when 'pending' + relation = relation.where(finished_at: nil) + .where.not(id: SolidQueue::FailedExecution.select(:job_id)) + .where.not(id: SolidQueue::ScheduledExecution.select(:job_id)) +``` + +In `filter_ready_jobs` (lines 116–133): +```ruby +if params[:class_name].present? + relation = relation.where(job_id: SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").select(:id)) +end +# ... +if params[:arguments].present? + relation = relation.where(job_id: SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").select(:id)) +end +``` + +Apply the same pattern to `filter_scheduled_jobs` (lines 135–152), `filter_failed_jobs` (lines 169–195). + +**Step 4: Fix `queues_controller.rb`** + +In `filter_queue_jobs` (lines 71–95): +```ruby +when 'failed' + relation = relation.where(id: SolidQueue::FailedExecution.select(:job_id)) +when 'scheduled' + relation = relation.where(id: SolidQueue::ScheduledExecution.select(:job_id)) +when 'pending' + relation = relation.where(id: SolidQueue::ReadyExecution.select(:job_id)) +when 'in_progress' + relation = relation.where(id: SolidQueue::ClaimedExecution.select(:job_id)) +``` + +**Step 5: Fix `in_progress_jobs_controller.rb`** + +In `filter_in_progress_jobs` (lines 21–35): +```ruby +if params[:class_name].present? + relation = relation.where(job_id: SolidQueue::Job.where('class_name LIKE ?', "%#{params[:class_name]}%").select(:id)) +end + +if params[:arguments].present? + relation = relation.where(job_id: SolidQueue::Job.where('arguments::text ILIKE ?', "%#{params[:arguments]}%").select(:id)) +end +``` + +**Step 6: Run the pluck spec** + +```bash +bundle exec rspec spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb -f doc +``` + +Expected: All PASS + +**Step 7: Run full suite** + +```bash +bundle exec rspec +``` + +Expected: All passing + +**Step 8: Commit** + +```bash +git add app/controllers/solid_queue_monitor/base_controller.rb \ + app/controllers/solid_queue_monitor/queues_controller.rb \ + app/controllers/solid_queue_monitor/in_progress_jobs_controller.rb \ + spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb +git commit -m "perf: replace all unbounded pluck calls with subqueries + +Every filter method was loading full ID arrays into Ruby via pluck(:id) +and pluck(:job_id), then passing them as WHERE IN (...) with potentially +millions of values. Replaced all 10+ instances across base_controller, +queues_controller, and in_progress_jobs_controller with select(:id) / +select(:job_id) subqueries that execute entirely in the database." +``` + +--- + +## Task 3: Eliminate N+1 in QueuesPresenter + +**Why:** `QueuesPresenter#generate_row` fires 3 COUNT queries per queue (ready, scheduled, failed). With 20 queues that's 60 queries. Fix by pre-aggregating in the controller with 3 GROUP BY queries total. + +**Files:** +- Modify: `app/controllers/solid_queue_monitor/queues_controller.rb` +- Modify: `app/presenters/solid_queue_monitor/queues_presenter.rb` +- Modify: `spec/requests/solid_queue_monitor/queues_spec.rb` + +--- + +**Step 1: Write a test** + +Add to `spec/requests/solid_queue_monitor/queues_spec.rb` inside the `GET /queues` describe: + +```ruby +it 'displays ready, scheduled, and failed counts per queue' do + create(:solid_queue_ready_execution, queue_name: 'default') + create(:solid_queue_scheduled_execution, queue_name: 'default') + create(:solid_queue_failed_execution) + + get '/queues' + + expect(response).to have_http_status(:ok) + # The counts should appear in the table + expect(response.body).to include('Ready Jobs') + expect(response.body).to include('Scheduled Jobs') + expect(response.body).to include('Failed Jobs') +end +``` + +**Step 2: Run to confirm it passes (existing behaviour baseline)** + +```bash +bundle exec rspec spec/requests/solid_queue_monitor/queues_spec.rb -f doc +``` + +Expected: PASS (we're verifying the output stays the same after refactor) + +**Step 3: Update `QueuesController#index`** + +```ruby +def index + base_query = SolidQueue::Job.group(:queue_name) + .select('queue_name, COUNT(*) as job_count') + @queues = apply_queue_sorting(base_query) + @paused_queues = QueuePauseService.paused_queues + @queue_stats = aggregate_queue_stats + + render_page('Queues', SolidQueueMonitor::QueuesPresenter.new( + @queues, @paused_queues, + queue_stats: @queue_stats, + sort: sort_params + ).render) +end +``` + +Add private method: + +```ruby +def aggregate_queue_stats + { + ready: SolidQueue::ReadyExecution.group(:queue_name).count, + scheduled: SolidQueue::ScheduledExecution.group(:queue_name).count, + failed: SolidQueue::FailedExecution.joins(:job) + .group('solid_queue_jobs.queue_name').count + } +end +``` + +**Step 4: Update QueuesPresenter** + +Change the initializer to accept `queue_stats`: + +```ruby +def initialize(records, paused_queues = [], sort: {}, queue_stats: {}) + @records = records + @paused_queues = paused_queues + @sort = sort + @queue_stats = queue_stats +end +``` + +Replace the per-row query methods with a single lookup: + +```ruby +def generate_row(queue) + queue_name = queue.queue_name || 'default' + paused = @paused_queues.include?(queue_name) + + <<-HTML + + #{queue_link(queue_name)} + #{status_badge(paused)} + #{queue.job_count} + #{@queue_stats.dig(:ready, queue_name) || 0} + #{@queue_stats.dig(:scheduled, queue_name) || 0} + #{@queue_stats.dig(:failed, queue_name) || 0} + #{action_button(queue_name, paused)} + + HTML +end +``` + +Remove the `ready_jobs_count`, `scheduled_jobs_count`, and `failed_jobs_count` methods entirely. + +**Step 5: Run tests** + +```bash +bundle exec rspec spec/requests/solid_queue_monitor/queues_spec.rb -f doc +``` + +Expected: All PASS + +**Step 6: Run full suite** + +```bash +bundle exec rspec +``` + +Expected: All passing + +**Step 7: Commit** + +```bash +git add app/controllers/solid_queue_monitor/queues_controller.rb \ + app/presenters/solid_queue_monitor/queues_presenter.rb \ + spec/requests/solid_queue_monitor/queues_spec.rb +git commit -m "perf: eliminate N+1 on queues index page + +QueuesPresenter fired 3 COUNT queries per queue row (ready, scheduled, +failed) — 60 queries for 20 queues. Now pre-aggregates with 3 GROUP BY +queries in the controller and passes the result hash to the presenter." +``` + +--- + +## Task 4: Fix ChartDataService memory explosion (cross-DB compatible) + +**Why:** `fetch_created_counts` does `pluck(:created_at)` loading potentially hundreds of thousands of timestamps into Ruby, then iterates O(N x buckets) to assign them. With a 24h window processing 1000 jobs/hour, that's 24K rows — manageable. But at scale or with wider windows (7d), this can blow up. + +**Approach:** Use SQL `GROUP BY` for bucketing but in a cross-DB way. Instead of PostgreSQL's `EXTRACT(EPOCH FROM ...)`, we do the grouping in the database using `COUNT` with a computed bucket key that works on both SQLite and PostgreSQL. The key insight: we can compute the bucket index `FLOOR((epoch - start_epoch) / interval)` using database-agnostic integer arithmetic on the primary key timestamp columns. However, SQLite lacks `EXTRACT(EPOCH FROM ...)`. + +**Simplest cross-DB approach:** Use the database for filtering and counting, Ruby only for bucket assignment — but on **counts per discrete timestamp** instead of raw timestamps. This is a middle ground that dramatically reduces data transfer without requiring DB-specific SQL. + +Actually, the cleanest approach: **group by a computed bucket in SQL, with an adapter-aware expression.** + +**Files:** +- Modify: `app/services/solid_queue_monitor/chart_data_service.rb` +- Modify: `spec/services/solid_queue_monitor/chart_data_service_spec.rb` + +--- + +**Step 1: Replace the spec with behaviour-based tests (no mocks)** + +Replace `spec/services/solid_queue_monitor/chart_data_service_spec.rb`: + +```ruby +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe SolidQueueMonitor::ChartDataService do + describe '#calculate' do + let(:service) { described_class.new(time_range: time_range) } + let(:time_range) { '1d' } + + context 'with no data' do + it 'returns the required keys' do + result = service.calculate + expect(result).to include(:labels, :created, :completed, :failed, + :totals, :time_range, :time_range_label, :available_ranges) + end + + it 'returns correct bucket count for 1d' do + result = service.calculate + expect(result[:labels].size).to eq(24) + expect(result[:created].size).to eq(24) + expect(result[:completed].size).to eq(24) + expect(result[:failed].size).to eq(24) + end + + it 'returns all zeros' do + result = service.calculate + expect(result[:totals]).to eq({ created: 0, completed: 0, failed: 0 }) + end + + it 'returns the current time range' do + expect(service.calculate[:time_range]).to eq('1d') + end + + it 'returns all available time ranges' do + expect(service.calculate[:available_ranges].keys).to eq(%w[15m 30m 1h 3h 6h 12h 1d 3d 1w]) + end + end + + context 'with 1h time range' do + let(:time_range) { '1h' } + it('returns 12 buckets') { expect(service.calculate[:labels].size).to eq(12) } + end + + context 'with 1w time range' do + let(:time_range) { '1w' } + it('returns 28 buckets') { expect(service.calculate[:labels].size).to eq(28) } + end + + context 'with invalid time range' do + let(:time_range) { 'invalid' } + + it 'defaults to 1d with 24 buckets' do + result = service.calculate + expect(result[:time_range]).to eq('1d') + expect(result[:labels].size).to eq(24) + end + end + + context 'with jobs in the time window' do + let(:time_range) { '1h' } + + before do + now = Time.current + create(:solid_queue_job, created_at: now - 10.minutes) + create(:solid_queue_job, created_at: now - 10.minutes) + create(:solid_queue_job, :completed, + created_at: now - 25.minutes, finished_at: now - 20.minutes) + create(:solid_queue_failed_execution, created_at: now - 15.minutes) + end + + it 'counts created jobs' do + # At least 2 regular + 1 completed + 1 from failed execution factory + expect(service.calculate[:created].sum).to be >= 2 + end + + it 'counts completed jobs' do + expect(service.calculate[:completed].sum).to eq(1) + end + + it 'counts failed executions' do + expect(service.calculate[:failed].sum).to eq(1) + end + + it 'totals match bucket sums' do + result = service.calculate + expect(result[:totals][:created]).to eq(result[:created].sum) + expect(result[:totals][:completed]).to eq(result[:completed].sum) + expect(result[:totals][:failed]).to eq(result[:failed].sum) + end + end + + context 'with jobs outside the window' do + let(:time_range) { '1h' } + before { create(:solid_queue_job, created_at: 2.hours.ago) } + + it 'excludes them' do + expect(service.calculate[:created].sum).to eq(0) + end + end + end + + describe 'constants' do + it 'defines all time ranges' do + expect(described_class::TIME_RANGES.keys).to eq(%w[15m 30m 1h 3h 6h 12h 1d 3d 1w]) + end + + it 'has required config per range' do + described_class::TIME_RANGES.each_value do |config| + expect(config).to include(:duration, :buckets, :label_format, :label) + end + end + + it 'defaults to 1d' do + expect(described_class::DEFAULT_TIME_RANGE).to eq('1d') + end + end +end +``` + +**Step 2: Run to establish baseline** + +```bash +bundle exec rspec spec/services/solid_queue_monitor/chart_data_service_spec.rb -f doc +``` + +Some tests will fail because the old code mocks pluck and the new tests hit the DB directly. + +**Step 3: Rewrite ChartDataService** + +The approach: use `COUNT` + `GROUP BY` with a bucket-index computation that works on both SQLite and PostgreSQL. + +- SQLite: `CAST((strftime('%s', column) - start_epoch) / interval AS INTEGER)` +- PostgreSQL: `CAST((EXTRACT(EPOCH FROM column) - start_epoch) / interval AS INTEGER)` + +We detect the adapter once and use the right expression. + +Replace `app/services/solid_queue_monitor/chart_data_service.rb`: + +```ruby +# frozen_string_literal: true + +module SolidQueueMonitor + class ChartDataService + TIME_RANGES = { + '15m' => { duration: 15.minutes, buckets: 15, label_format: '%H:%M', label: 'Last 15 minutes' }, + '30m' => { duration: 30.minutes, buckets: 15, label_format: '%H:%M', label: 'Last 30 minutes' }, + '1h' => { duration: 1.hour, buckets: 12, label_format: '%H:%M', label: 'Last 1 hour' }, + '3h' => { duration: 3.hours, buckets: 18, label_format: '%H:%M', label: 'Last 3 hours' }, + '6h' => { duration: 6.hours, buckets: 24, label_format: '%H:%M', label: 'Last 6 hours' }, + '12h' => { duration: 12.hours, buckets: 24, label_format: '%H:%M', label: 'Last 12 hours' }, + '1d' => { duration: 1.day, buckets: 24, label_format: '%H:%M', label: 'Last 24 hours' }, + '3d' => { duration: 3.days, buckets: 36, label_format: '%m/%d %H:%M', label: 'Last 3 days' }, + '1w' => { duration: 7.days, buckets: 28, label_format: '%m/%d', label: 'Last 7 days' } + }.freeze + + DEFAULT_TIME_RANGE = '1d' + + def initialize(time_range: DEFAULT_TIME_RANGE) + @time_range = TIME_RANGES.key?(time_range) ? time_range : DEFAULT_TIME_RANGE + @config = TIME_RANGES[@time_range] + end + + def calculate + end_time = Time.current + start_time = end_time - @config[:duration] + bucket_seconds = (@config[:duration] / @config[:buckets]).to_i + buckets = build_buckets(start_time, bucket_seconds) + + created_data = bucket_counts(SolidQueue::Job, :created_at, start_time, end_time, bucket_seconds) + completed_data = bucket_counts(SolidQueue::Job, :finished_at, start_time, end_time, bucket_seconds, exclude_nil: true) + failed_data = bucket_counts(SolidQueue::FailedExecution, :created_at, start_time, end_time, bucket_seconds) + + created_arr = fill_buckets(buckets, created_data) + completed_arr = fill_buckets(buckets, completed_data) + failed_arr = fill_buckets(buckets, failed_data) + + { + labels: buckets.map { |b| b[:label] }, # rubocop:disable Rails/Pluck + created: created_arr, + completed: completed_arr, + failed: failed_arr, + totals: { created: created_arr.sum, completed: completed_arr.sum, failed: failed_arr.sum }, + time_range: @time_range, + time_range_label: @config[:label], + available_ranges: TIME_RANGES.transform_values { |v| v[:label] } + } + end + + private + + def build_buckets(start_time, bucket_seconds) + @config[:buckets].times.map do |i| + bucket_start = start_time + (i * bucket_seconds) + { index: i, start: bucket_start, label: bucket_start.strftime(@config[:label_format]) } + end + end + + # Returns a Hash of { bucket_index => count } using SQL GROUP BY. + # The bucket index is computed as: (epoch(column) - epoch(start_time)) / interval + # This works identically on PostgreSQL and SQLite. + def bucket_counts(model, column, start_time, end_time, interval, exclude_nil: false) + start_epoch = start_time.to_i + expr = bucket_index_expr(column, start_epoch, interval) + + scope = model.where(column => start_time..end_time) + scope = scope.where.not(column => nil) if exclude_nil + + scope + .group(Arel.sql(expr)) + .pluck(Arel.sql("#{expr} AS bucket_idx, COUNT(*) AS cnt")) + .to_h { |idx, cnt| [idx.to_i, cnt] } + end + + def fill_buckets(buckets, index_counts) + buckets.map { |b| index_counts.fetch(b[:index], 0) } + end + + # Cross-DB bucket index expression. + # PostgreSQL: CAST((EXTRACT(EPOCH FROM col) - start) / interval AS INTEGER) + # SQLite: CAST((CAST(strftime('%s', col) AS INTEGER) - start) / interval AS INTEGER) + def bucket_index_expr(column, start_epoch, interval_seconds) + if sqlite? + "CAST((CAST(strftime('%s', #{column}) AS INTEGER) - #{start_epoch}) / #{interval_seconds} AS INTEGER)" + else + "CAST((EXTRACT(EPOCH FROM #{column}) - #{start_epoch}) / #{interval_seconds} AS INTEGER)" + end + end + + def sqlite? + ActiveRecord::Base.connection.adapter_name.downcase.include?('sqlite') + end + end +end +``` + +**Step 4: Run the spec** + +```bash +bundle exec rspec spec/services/solid_queue_monitor/chart_data_service_spec.rb -f doc +``` + +Expected: All PASS + +**Step 5: Run full suite** + +```bash +bundle exec rspec +``` + +Expected: All passing + +**Step 6: Commit** + +```bash +git add app/services/solid_queue_monitor/chart_data_service.rb \ + spec/services/solid_queue_monitor/chart_data_service_spec.rb +git commit -m "perf: replace in-memory chart bucketing with SQL GROUP BY + +ChartDataService was plucking all matching timestamps into Ruby and +bucketing them in O(N x buckets) loops. Now uses SQL GROUP BY with a +computed bucket index — returns at most N bucket rows from the DB. + +Uses adapter-aware SQL: EXTRACT(EPOCH FROM ...) for PostgreSQL, +strftime('%s', ...) for SQLite. Tests pass on both." +``` + +--- + +## Task 5: Add `config.show_chart` to disable chart on overview + +**Why:** Even with SQL GROUP BY, the chart fires 3 queries on every overview load. Some teams don't use the visualisation and want zero overhead. This is the only config flag we add — it's a genuine feature toggle (show/hide UI), not a performance workaround. + +**Files:** +- Modify: `lib/solid_queue_monitor.rb` +- Modify: `lib/generators/solid_queue_monitor/templates/initializer.rb` +- Modify: `app/controllers/solid_queue_monitor/overview_controller.rb` +- Modify: `spec/requests/solid_queue_monitor/overview_spec.rb` + +--- + +**Step 1: Add config attribute** + +In `lib/solid_queue_monitor.rb`, add `show_chart` to the attr_accessor and set default: + +```ruby +attr_accessor :username, :password, :jobs_per_page, :authentication_enabled, + :auto_refresh_enabled, :auto_refresh_interval, :show_chart + +@show_chart = true +``` + +**Step 2: Write the test** + +Add to `spec/requests/solid_queue_monitor/overview_spec.rb`: + +```ruby +context 'with chart disabled' do + around do |example| + original = SolidQueueMonitor.show_chart + SolidQueueMonitor.show_chart = false + example.run + SolidQueueMonitor.show_chart = original + end + + it 'does not call ChartDataService' do + expect(SolidQueueMonitor::ChartDataService).not_to receive(:new) + get '/' + expect(response).to have_http_status(:ok) + end + + it 'does not render chart section' do + get '/' + expect(response.body).not_to include('chart-section') + end +end +``` + +**Step 3: Run to confirm failure** + +```bash +bundle exec rspec spec/requests/solid_queue_monitor/overview_spec.rb \ + -e 'chart disabled' -f doc +``` + +Expected: FAIL — ChartDataService is still called + +**Step 4: Update OverviewController** + +```ruby +def index + @stats = SolidQueueMonitor::StatsCalculator.calculate + @chart_data = SolidQueueMonitor.show_chart ? SolidQueueMonitor::ChartDataService.new(time_range: time_range_param).calculate : nil + + recent_jobs_query = SolidQueue::Job.limit(100) + sorted_query = apply_sorting(filter_jobs(recent_jobs_query), SORTABLE_COLUMNS, 'created_at', :desc) + @recent_jobs = paginate(sorted_query) + + preload_job_statuses(@recent_jobs[:records]) + + render_page('Overview', generate_overview_content) +end + +private + +def generate_overview_content + html = SolidQueueMonitor::StatsPresenter.new(@stats).render + html += SolidQueueMonitor::ChartPresenter.new(@chart_data).render if @chart_data + html + SolidQueueMonitor::JobsPresenter.new(@recent_jobs[:records], + current_page: @recent_jobs[:current_page], + total_pages: @recent_jobs[:total_pages], + filters: filter_params, + sort: sort_params).render +end +``` + +**Step 5: Update initializer template** + +Add to `lib/generators/solid_queue_monitor/templates/initializer.rb`: + +```ruby + # Disable the chart on the overview page to skip chart queries entirely. + # config.show_chart = true +``` + +**Step 6: Run tests** + +```bash +bundle exec rspec spec/requests/solid_queue_monitor/overview_spec.rb -f doc +``` + +Expected: All PASS + +**Step 7: Run full suite** + +```bash +bundle exec rspec +``` + +Expected: All passing + +**Step 8: Commit** + +```bash +git add lib/solid_queue_monitor.rb \ + lib/generators/solid_queue_monitor/templates/initializer.rb \ + app/controllers/solid_queue_monitor/overview_controller.rb \ + spec/requests/solid_queue_monitor/overview_spec.rb +git commit -m "feat: add config.show_chart to disable chart on overview + +Adds config.show_chart (default: true). When false, skips +ChartDataService and chart rendering entirely — zero additional +queries on the overview page." +``` + +--- + +## Task 6: Update docs, CHANGELOG, ROADMAP + +**Files:** +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `ROADMAP.md` + +**Step 1: Add to README** + +After the configuration section, add: + +```markdown +### Performance at Scale + +The monitor is designed to work efficiently with large datasets (millions of jobs). +Overview stats are derived entirely from Solid Queue's execution tables — no expensive +COUNT queries on the jobs table. + +If you don't need the chart visualisation, you can disable it to skip those queries: + +```ruby +SolidQueueMonitor.setup do |config| + config.show_chart = false +end +``` +``` + +**Step 2: Add to CHANGELOG** + +```markdown +## [Unreleased] + +### Performance +- **Breaking:** Overview stats now show "Active Jobs" (ready + scheduled + in_progress + failed) instead of "Total Jobs" and "Completed". This eliminates 3 full-table COUNT queries on solid_queue_jobs that caused gateway timeouts at scale (52s each at 4M rows). +- Fix: All status filter queries now use SQL subqueries instead of loading IDs into Ruby memory via `pluck`. +- Fix: Queues index page now pre-aggregates ready/scheduled/failed counts with 3 GROUP BY queries instead of 3 COUNT queries per queue row (N+1 elimination). +- Fix: ChartDataService uses SQL GROUP BY for time bucketing instead of plucking all timestamps into memory. Works on both PostgreSQL and SQLite. +- Add: `config.show_chart` (default: true) — set to false to disable chart queries entirely on the overview page. +``` + +**Step 3: Update ROADMAP** + +Add to Medium Priority table: +```markdown +| Large Dataset Performance | Execution-table-only stats, N+1 fixes, SQL chart bucketing, optional chart | Done | +``` + +**Step 4: Commit** + +```bash +git add README.md CHANGELOG.md ROADMAP.md +git commit -m "docs: document performance improvements for large datasets" +``` + +--- + +## Final Verification + +```bash +bundle exec rspec --format progress +``` + +All green. + +--- + +## Summary: Issue #27 Points Addressed + +| Issue Point | Resolution | +|-------------|------------| +| COUNT(*) on 4M jobs causes 52s timeout | **Eliminated entirely.** StatsCalculator no longer queries solid_queue_jobs. Stats derived from execution tables. | +| Chart aggregation queries are slow | SQL GROUP BY (cross-DB). Optional `show_chart = false` to skip entirely. | +| Queue page N+1 counters | Pre-aggregated with 3 GROUP BY queries regardless of queue count. | +| Expensive total counts for pagination | **Not needed.** Pagination never hits the jobs table at scale — overview is capped at 100, other pages paginate small execution tables. | +| Default ordering on large tables | Execution tables are small; ordering by `created_at` is fine. The jobs table was the problem, and we no longer COUNT it. | + +## What We Deliberately Did NOT Add + +| Rejected Approach | Why | +|-------------------|-----| +| `config.approximate_counts` / `pg_class.reltuples` | PostgreSQL-only. Tests run on SQLite. Problem eliminated by not counting the jobs table. | +| `config.paginate_without_count` | Pagination COUNT is only expensive on jobs table. Overview is capped at 100 records. Other pages paginate small execution tables. | +| `config.root_redirect_to` | Workaround for a slow overview. If overview is fast, redirect is pointless. | +| Raw PostgreSQL SQL for chart bucketing | Breaks SQLite test suite. Used adapter-aware SQL instead. | From c8c35b097743e2390c791e0d6bb7f1d93f83bd49 Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Sat, 7 Mar 2026 10:54:16 +0530 Subject: [PATCH 6/7] =?UTF-8?q?release:=20v1.2.0=20=E2=80=94=20Performance?= =?UTF-8?q?=20at=20Scale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump version to 1.2.0, set CHANGELOG release date, update README gem version reference. --- CHANGELOG.md | 2 +- README.md | 2 +- lib/solid_queue_monitor/version.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e346e8f..1a6968c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Unreleased] +## [1.2.0] - 2026-03-07 ### Changed diff --git a/README.md b/README.md index 4d448da..2f78ba0 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ A lightweight, zero-dependency web interface for monitoring Solid Queue backgrou Add this line to your application's Gemfile: ```ruby -gem 'solid_queue_monitor', '~> 1.1' +gem 'solid_queue_monitor', '~> 1.2' ``` Then execute: diff --git a/lib/solid_queue_monitor/version.rb b/lib/solid_queue_monitor/version.rb index 7dac272..82420dd 100644 --- a/lib/solid_queue_monitor/version.rb +++ b/lib/solid_queue_monitor/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SolidQueueMonitor - VERSION = '1.1.0' + VERSION = '1.2.0' end From 0fee84fb7ad220033f9762c647e56361b0e9ab96 Mon Sep 17 00:00:00 2001 From: Vishal Sadriya Date: Sat, 7 Mar 2026 10:59:43 +0530 Subject: [PATCH 7/7] fix: resolve rubocop and CI lint offenses - Fix RSpec/MessageSpies: use have_received instead of receive - Fix Layout/HashAlignment, Layout/ArgumentAlignment auto-corrections - Fix Style/ConditionalAssignment in base_controller - Disable false-positive Style/HashTransformKeys (pluck returns Array) - Disable RSpec/DescribeClass for source-scanning spec - Update Gemfile.lock with version 1.2.0 --- Gemfile.lock | 2 +- .../solid_queue_monitor/base_controller.rb | 10 +++---- .../overview_controller.rb | 8 +++--- .../solid_queue_monitor/queues_controller.rb | 6 ++-- .../solid_queue_monitor/chart_data_service.rb | 28 ++++++++++--------- .../solid_queue_monitor/stats_calculator.rb | 8 +++--- .../solid_queue_monitor/overview_spec.rb | 3 +- .../chart_data_service_spec.rb | 3 ++ .../no_unbounded_pluck_spec.rb | 8 +++--- .../stats_calculator_spec.rb | 3 +- 10 files changed, 43 insertions(+), 36 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index cb0ff01..2932e71 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - solid_queue_monitor (1.1.0) + solid_queue_monitor (1.2.0) rails (>= 7.0) solid_queue (>= 0.1.0) diff --git a/app/controllers/solid_queue_monitor/base_controller.rb b/app/controllers/solid_queue_monitor/base_controller.rb index 87acb8f..7178aa7 100644 --- a/app/controllers/solid_queue_monitor/base_controller.rb +++ b/app/controllers/solid_queue_monitor/base_controller.rb @@ -164,11 +164,11 @@ def filter_failed_jobs(relation) end if params[:queue_name].present? - if relation.column_names.include?('queue_name') - relation = relation.where('queue_name LIKE ?', "%#{params[:queue_name]}%") - else - relation = relation.where(job_id: SolidQueue::Job.where('queue_name LIKE ?', "%#{params[:queue_name]}%").select(:id)) - end + relation = if relation.column_names.include?('queue_name') + relation.where('queue_name LIKE ?', "%#{params[:queue_name]}%") + else + relation.where(job_id: SolidQueue::Job.where('queue_name LIKE ?', "%#{params[:queue_name]}%").select(:id)) + end end if params[:arguments].present? diff --git a/app/controllers/solid_queue_monitor/overview_controller.rb b/app/controllers/solid_queue_monitor/overview_controller.rb index 2307134..b892db8 100644 --- a/app/controllers/solid_queue_monitor/overview_controller.rb +++ b/app/controllers/solid_queue_monitor/overview_controller.rb @@ -32,10 +32,10 @@ def generate_overview_content html = SolidQueueMonitor::StatsPresenter.new(@stats).render html += SolidQueueMonitor::ChartPresenter.new(@chart_data).render if @chart_data html + SolidQueueMonitor::JobsPresenter.new(@recent_jobs[:records], - current_page: @recent_jobs[:current_page], - total_pages: @recent_jobs[:total_pages], - filters: filter_params, - sort: sort_params).render + current_page: @recent_jobs[:current_page], + total_pages: @recent_jobs[:total_pages], + filters: filter_params, + sort: sort_params).render end end end diff --git a/app/controllers/solid_queue_monitor/queues_controller.rb b/app/controllers/solid_queue_monitor/queues_controller.rb index eed52ac..b420537 100644 --- a/app/controllers/solid_queue_monitor/queues_controller.rb +++ b/app/controllers/solid_queue_monitor/queues_controller.rb @@ -64,10 +64,10 @@ def resume def aggregate_queue_stats { - ready: SolidQueue::ReadyExecution.group(:queue_name).count, + ready: SolidQueue::ReadyExecution.group(:queue_name).count, scheduled: SolidQueue::ScheduledExecution.group(:queue_name).count, - failed: SolidQueue::FailedExecution.joins(:job) - .group('solid_queue_jobs.queue_name').count + failed: SolidQueue::FailedExecution.joins(:job) + .group('solid_queue_jobs.queue_name').count } end diff --git a/app/services/solid_queue_monitor/chart_data_service.rb b/app/services/solid_queue_monitor/chart_data_service.rb index b1035ef..6b9fce6 100644 --- a/app/services/solid_queue_monitor/chart_data_service.rb +++ b/app/services/solid_queue_monitor/chart_data_service.rb @@ -5,13 +5,13 @@ class ChartDataService TIME_RANGES = { '15m' => { duration: 15.minutes, buckets: 15, label_format: '%H:%M', label: 'Last 15 minutes' }, '30m' => { duration: 30.minutes, buckets: 15, label_format: '%H:%M', label: 'Last 30 minutes' }, - '1h' => { duration: 1.hour, buckets: 12, label_format: '%H:%M', label: 'Last 1 hour' }, - '3h' => { duration: 3.hours, buckets: 18, label_format: '%H:%M', label: 'Last 3 hours' }, - '6h' => { duration: 6.hours, buckets: 24, label_format: '%H:%M', label: 'Last 6 hours' }, - '12h' => { duration: 12.hours, buckets: 24, label_format: '%H:%M', label: 'Last 12 hours' }, - '1d' => { duration: 1.day, buckets: 24, label_format: '%H:%M', label: 'Last 24 hours' }, - '3d' => { duration: 3.days, buckets: 36, label_format: '%m/%d %H:%M', label: 'Last 3 days' }, - '1w' => { duration: 7.days, buckets: 28, label_format: '%m/%d', label: 'Last 7 days' } + '1h' => { duration: 1.hour, buckets: 12, label_format: '%H:%M', label: 'Last 1 hour' }, + '3h' => { duration: 3.hours, buckets: 18, label_format: '%H:%M', label: 'Last 3 hours' }, + '6h' => { duration: 6.hours, buckets: 24, label_format: '%H:%M', label: 'Last 6 hours' }, + '12h' => { duration: 12.hours, buckets: 24, label_format: '%H:%M', label: 'Last 12 hours' }, + '1d' => { duration: 1.day, buckets: 24, label_format: '%H:%M', label: 'Last 24 hours' }, + '3d' => { duration: 3.days, buckets: 36, label_format: '%m/%d %H:%M', label: 'Last 3 days' }, + '1w' => { duration: 7.days, buckets: 28, label_format: '%m/%d', label: 'Last 7 days' } }.freeze DEFAULT_TIME_RANGE = '1d' @@ -36,12 +36,12 @@ def calculate failed_arr = fill_buckets(buckets, failed_data) { - labels: buckets.map { |b| b[:label] }, # rubocop:disable Rails/Pluck - created: created_arr, - completed: completed_arr, - failed: failed_arr, - totals: { created: created_arr.sum, completed: completed_arr.sum, failed: failed_arr.sum }, - time_range: @time_range, + labels: buckets.map { |b| b[:label] }, # rubocop:disable Rails/Pluck + created: created_arr, + completed: completed_arr, + failed: failed_arr, + totals: { created: created_arr.sum, completed: completed_arr.sum, failed: failed_arr.sum }, + time_range: @time_range, time_range_label: @config[:label], available_ranges: TIME_RANGES.transform_values { |v| v[:label] } } @@ -66,10 +66,12 @@ def bucket_counts(model, column, start_time, end_time, interval, exclude_nil: fa scope = model.where(column => start_time..end_time) scope = scope.where.not(column => nil) if exclude_nil + # rubocop:disable Style/HashTransformKeys -- pluck returns Array, not Hash scope .group(Arel.sql(expr)) .pluck(Arel.sql("#{expr} AS bucket_idx, COUNT(*) AS cnt")) .to_h { |idx, cnt| [idx.to_i, cnt] } + # rubocop:enable Style/HashTransformKeys end def fill_buckets(buckets, index_counts) diff --git a/app/services/solid_queue_monitor/stats_calculator.rb b/app/services/solid_queue_monitor/stats_calculator.rb index b7c9aa9..60b467d 100644 --- a/app/services/solid_queue_monitor/stats_calculator.rb +++ b/app/services/solid_queue_monitor/stats_calculator.rb @@ -11,11 +11,11 @@ def self.calculate { active_jobs: ready + scheduled + in_progress + failed, - scheduled: scheduled, - ready: ready, - failed: failed, + scheduled: scheduled, + ready: ready, + failed: failed, in_progress: in_progress, - recurring: recurring + recurring: recurring } end end diff --git a/spec/requests/solid_queue_monitor/overview_spec.rb b/spec/requests/solid_queue_monitor/overview_spec.rb index 807bfa2..5e5089a 100644 --- a/spec/requests/solid_queue_monitor/overview_spec.rb +++ b/spec/requests/solid_queue_monitor/overview_spec.rb @@ -50,9 +50,10 @@ end it 'does not call ChartDataService' do - expect(SolidQueueMonitor::ChartDataService).not_to receive(:new) + allow(SolidQueueMonitor::ChartDataService).to receive(:new).and_call_original get '/' expect(response).to have_http_status(:ok) + expect(SolidQueueMonitor::ChartDataService).not_to have_received(:new) end it 'does not render chart section' do diff --git a/spec/services/solid_queue_monitor/chart_data_service_spec.rb b/spec/services/solid_queue_monitor/chart_data_service_spec.rb index f831e6f..2ef091a 100644 --- a/spec/services/solid_queue_monitor/chart_data_service_spec.rb +++ b/spec/services/solid_queue_monitor/chart_data_service_spec.rb @@ -38,11 +38,13 @@ context 'with 1h time range' do let(:time_range) { '1h' } + it('returns 12 buckets') { expect(service.calculate[:labels].size).to eq(12) } end context 'with 1w time range' do let(:time_range) { '1w' } + it('returns 28 buckets') { expect(service.calculate[:labels].size).to eq(28) } end @@ -91,6 +93,7 @@ context 'with jobs outside the window' do let(:time_range) { '1h' } + before { create(:solid_queue_job, created_at: 2.hours.ago) } it 'excludes them' do diff --git a/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb b/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb index 697fd88..27b1d34 100644 --- a/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb +++ b/spec/services/solid_queue_monitor/no_unbounded_pluck_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe 'No unbounded pluck calls in controllers' do +RSpec.describe 'No unbounded pluck calls in controllers' do # rubocop:disable RSpec/DescribeClass root = File.expand_path('../../..', __dir__) controller_files = Dir[File.join(root, 'app', 'controllers', '**', '*.rb')] @@ -27,9 +27,9 @@ end expect(pluck_filter_lines).to be_empty, - "Found unbounded pluck calls used for filtering in #{relative}:\n" \ - "#{pluck_filter_lines.map(&:strip).join("\n")}\n" \ - "Use .select(:job_id) or .select(:id) for subqueries instead." + "Found unbounded pluck calls used for filtering in #{relative}:\n" \ + "#{pluck_filter_lines.map(&:strip).join("\n")}\n" \ + 'Use .select(:job_id) or .select(:id) for subqueries instead.' end end end diff --git a/spec/services/solid_queue_monitor/stats_calculator_spec.rb b/spec/services/solid_queue_monitor/stats_calculator_spec.rb index 8214751..9e4d89c 100644 --- a/spec/services/solid_queue_monitor/stats_calculator_spec.rb +++ b/spec/services/solid_queue_monitor/stats_calculator_spec.rb @@ -42,8 +42,9 @@ end it 'does not query the jobs table for counts' do - expect(SolidQueue::Job).not_to receive(:count) + allow(SolidQueue::Job).to receive(:count).and_call_original described_class.calculate + expect(SolidQueue::Job).not_to have_received(:count) end end end