From e1d25de3c611df11aab34ba0a0118fc2ec984327 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 24 Apr 2025 11:54:31 +0300 Subject: [PATCH 01/19] Feature: Dynamic scheduled tasks --- README.md | 2 + app/models/solid_queue/recurring_task.rb | 1 + lib/solid_queue/configuration.rb | 10 +- lib/solid_queue/scheduler.rb | 6 ++ .../scheduler/recurring_schedule.rb | 46 ++++++-- test/unit/configuration_test.rb | 6 +- test/unit/scheduler_test.rb | 100 +++++++++++++++++- 7 files changed, 152 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b49f9c60..7c13bda3 100644 --- a/README.md +++ b/README.md @@ -707,6 +707,8 @@ Rails.application.config.after_initialize do # or to_prepare end ``` +You can also dynamically add or remove recurring tasks by creating or deleting SolidQueue::RecurringTask records. It works the same way as with static tasks, except you must set the static field to false. Changes won’t be picked up immediately — they take effect after about a one-minute delay. + It's possible to run multiple schedulers with the same `recurring_tasks` configuration, for example, if you have multiple servers for redundancy, and you run the `scheduler` in more than one of them. To avoid enqueuing duplicate tasks at the same time, an entry in a new `solid_queue_recurring_executions` table is created in the same transaction as the job is enqueued. This table has a unique index on `task_key` and `run_at`, ensuring only one entry per task per time will be created. This only works if you have `preserve_finished_jobs` set to `true` (the default), and the guarantee applies as long as you keep the jobs around. **Note**: a single recurring schedule is supported, so you can have multiple schedulers using the same schedule, but not multiple schedulers using different configurations. diff --git a/app/models/solid_queue/recurring_task.rb b/app/models/solid_queue/recurring_task.rb index 2a776c8a..5de7775f 100644 --- a/app/models/solid_queue/recurring_task.rb +++ b/app/models/solid_queue/recurring_task.rb @@ -11,6 +11,7 @@ class RecurringTask < Record validate :existing_job_class scope :static, -> { where(static: true) } + scope :dynamic, -> { where(static: false) } has_many :recurring_executions, foreign_key: :task_key, primary_key: :key diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 94169ca7..da6b0df4 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -103,7 +103,7 @@ def default_options end def invalid_tasks - recurring_tasks.select(&:invalid?) + static_recurring_tasks.select(&:invalid?) end def only_work? @@ -137,8 +137,8 @@ def dispatchers end def schedulers - if !skip_recurring_tasks? && recurring_tasks.any? - [ Process.new(:scheduler, recurring_tasks: recurring_tasks) ] + if !skip_recurring_tasks? + [ Process.new(:scheduler, recurring_tasks: static_recurring_tasks) ] else [] end @@ -154,8 +154,8 @@ def dispatchers_options .map { |options| options.dup.symbolize_keys } end - def recurring_tasks - @recurring_tasks ||= recurring_tasks_config.map do |id, options| + def static_recurring_tasks + @static_recurring_tasks ||= recurring_tasks_config.map do |id, options| RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule) end.compact end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 3cec90fa..3ac78d74 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -30,6 +30,12 @@ def run loop do break if shutting_down? + recurring_schedule.update_scheduled_tasks.tap do |updated_tasks| + if updated_tasks.any? + process.update_columns(metadata: metadata.compact) + end + end + interruptible_sleep(SLEEP_INTERVAL) end ensure diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index b765edf1..ea55f9f8 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,10 +4,11 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :configured_tasks, :scheduled_tasks + attr_reader :static_tasks, :configured_tasks, :scheduled_tasks def initialize(tasks) - @configured_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) + @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) + @configured_tasks = @static_tasks + dynamic_tasks @scheduled_tasks = Concurrent::Hash.new end @@ -17,8 +18,8 @@ def empty? def schedule_tasks wrap_in_app_executor do - persist_tasks - reload_tasks + persist_static_tasks + reload_static_tasks end configured_tasks.each do |task| @@ -26,6 +27,27 @@ def schedule_tasks end end + def dynamic_tasks + SolidQueue::RecurringTask.dynamic + end + + def schedule_new_dynamic_tasks + dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| + schedule_task(task) + end + end + + def unschedule_old_dynamic_tasks + (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| + scheduled_tasks[key].cancel + scheduled_tasks.delete(key) + end + end + + def update_scheduled_tasks + schedule_new_dynamic_tasks + unschedule_old_dynamic_tasks + end + def schedule_task(task) scheduled_tasks[task.key] = schedule(task) end @@ -35,18 +57,22 @@ def unschedule_tasks scheduled_tasks.clear end + def static_task_keys + static_tasks.map(&:key) + end + def task_keys - configured_tasks.map(&:key) + static_task_keys + dynamic_tasks.map(&:key) end private - def persist_tasks - SolidQueue::RecurringTask.static.where.not(key: task_keys).delete_all - SolidQueue::RecurringTask.create_or_update_all configured_tasks + def persist_static_tasks + SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all + SolidQueue::RecurringTask.create_or_update_all static_tasks end - def reload_tasks - @configured_tasks = SolidQueue::RecurringTask.where(key: task_keys).to_a + def reload_static_tasks + @static_tasks = SolidQueue::RecurringTask.static.where(key: static_task_keys).to_a end def schedule(task) diff --git a/test/unit/configuration_test.rb b/test/unit/configuration_test.rb index 11c2a5ff..14db95cb 100644 --- a/test/unit/configuration_test.rb +++ b/test/unit/configuration_test.rb @@ -21,7 +21,7 @@ class ConfigurationTest < ActiveSupport::TestCase test "default configuration when config given is empty" do configuration = SolidQueue::Configuration.new(config_file: config_file_path(:empty_configuration), recurring_schedule_file: config_file_path(:empty_configuration)) - assert_equal 2, configuration.configured_processes.count + assert_equal 3, configuration.configured_processes.count # includes scheduler for dynamic tasks assert_processes configuration, :worker, 1, queues: "*" assert_processes configuration, :dispatcher, 1, batch_size: SolidQueue::Configuration::DISPATCHER_DEFAULTS[:batch_size] end @@ -134,12 +134,12 @@ class ConfigurationTest < ActiveSupport::TestCase configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_production_only)) assert configuration.valid? - assert_processes configuration, :scheduler, 0 + assert_processes configuration, :scheduler, 1 # Starts in case of dynamic tasks assert_output(/Provided configuration file '[^']+' does not exist\./) do configuration = SolidQueue::Configuration.new(recurring_schedule_file: config_file_path(:recurring_with_empty)) assert configuration.valid? - assert_processes configuration, :scheduler, 0 + assert_processes configuration, :scheduler, 1 end # No processes diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 3e838c50..21f50f34 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -3,7 +3,7 @@ class SchedulerTest < ActiveSupport::TestCase self.use_transactional_tests = false - test "recurring schedule" do + test "recurring schedule (only static)" do recurring_tasks = { example_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks).tap(&:start) @@ -17,6 +17,41 @@ class SchedulerTest < ActiveSupport::TestCase scheduler.stop end + test "recurring schedule (only dynamic)" do + SolidQueue::RecurringTask.create( + key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_equal "Scheduler", process.kind + + assert_metadata process, recurring_schedule: [ "dynamic_task" ] + ensure + scheduler.stop + end + + test "recurring schedule (static + dynamic)" do + SolidQueue::RecurringTask.create( + key: "dynamic_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + + recurring_tasks = { static_task: { class: "AddToBufferJob", schedule: "every hour", args: 42 } } + + scheduler = SolidQueue::Scheduler.new(recurring_tasks: recurring_tasks).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + assert_equal "Scheduler", process.kind + + assert_metadata process, recurring_schedule: [ "static_task", "dynamic_task" ] + ensure + scheduler.stop + end + test "run more than one instance of the scheduler with recurring tasks" do recurring_tasks = { example_task: { class: "AddToBufferJob", schedule: "every second", args: 42 } } schedulers = 2.times.collect do @@ -37,4 +72,67 @@ class SchedulerTest < ActiveSupport::TestCase end end end + + test "updates metadata after adding dynamic task post-start" do + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| + s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } + s.start + end + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + # initially there are no recurring_schedule keys + assert process.metadata, {} + + # now create a dynamic task after the scheduler has booted + SolidQueue::RecurringTask.create( + key: "new_dynamic_task", + static: false, + class_name: "AddToBufferJob", + schedule: "every second", + arguments: [ 42 ] + ) + + sleep 1 + + process.reload + + # metadata should now include the new key + assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] + ensure + scheduler&.stop + end + + test "updates metadata after removing dynamic task post-start" do + old_dynamic_task = SolidQueue::RecurringTask.create( + key: "old_dynamic_task", + static: false, + class_name: "AddToBufferJob", + schedule: "every second", + arguments: [ 42 ] + ) + + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| + s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } + s.start + end + + wait_for_registered_processes(1, timeout: 1.second) + + process = SolidQueue::Process.first + # initially there is one recurring_schedule key + assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] + + old_dynamic_task.destroy + + sleep 1 + + process.reload + + # The task is unschedule after it's being removed, and it's reflected in the metadata + assert process.metadata, {} + ensure + scheduler&.stop + end end From 22899516799a0021c8fe6b143b96c02e84141474 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 19 Jun 2025 15:27:44 +0300 Subject: [PATCH 02/19] Extract polling_interval to scheduler configuration --- .../install/templates/config/queue.yml | 2 ++ lib/solid_queue/configuration.rb | 32 ++++++++++++------- lib/solid_queue/scheduler.rb | 8 ++--- test/unit/scheduler_test.rb | 10 ++---- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/lib/generators/solid_queue/install/templates/config/queue.yml b/lib/generators/solid_queue/install/templates/config/queue.yml index 15691e9d..d7b0e6b9 100644 --- a/lib/generators/solid_queue/install/templates/config/queue.yml +++ b/lib/generators/solid_queue/install/templates/config/queue.yml @@ -7,6 +7,8 @@ default: &default threads: 3 processes: <%%= ENV.fetch("JOB_CONCURRENCY", 1) %> polling_interval: 0.1 + scheduler: + polling_interval: 1 development: <<: *default diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index da6b0df4..244444bd 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -28,6 +28,10 @@ def instantiate concurrency_maintenance_interval: 600 } + SCHEDULER_DEFAULTS = { + polling_interval: 1 + } + DEFAULT_CONFIG_FILE_PATH = "config/queue.yml" DEFAULT_RECURRING_SCHEDULE_FILE_PATH = "config/recurring.yml" @@ -103,7 +107,7 @@ def default_options end def invalid_tasks - static_recurring_tasks.select(&:invalid?) + recurring_tasks.select(&:invalid?) end def only_work? @@ -137,11 +141,9 @@ def dispatchers end def schedulers - if !skip_recurring_tasks? - [ Process.new(:scheduler, recurring_tasks: static_recurring_tasks) ] - else - [] - end + return [] if skip_recurring_tasks? + + [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] end def workers_options @@ -154,17 +156,25 @@ def dispatchers_options .map { |options| options.dup.symbolize_keys } end - def static_recurring_tasks - @static_recurring_tasks ||= recurring_tasks_config.map do |id, options| + def scheduler_options + @scheduler_options ||= processes_config.fetch(:scheduler, {}).dup.symbolize_keys + end + + def recurring_tasks + @recurring_tasks ||= recurring_tasks_config.map do |id, options| RecurringTask.from_configuration(id, **options) if options&.has_key?(:schedule) end.compact end def processes_config @processes_config ||= config_from \ - options.slice(:workers, :dispatchers).presence || options[:config_file], - keys: [ :workers, :dispatchers ], - fallback: { workers: [ WORKER_DEFAULTS ], dispatchers: [ DISPATCHER_DEFAULTS ] } + options.slice(:workers, :dispatchers, :scheduler).presence || options[:config_file], + keys: [ :workers, :dispatchers, :scheduler ], + fallback: { + workers: [ WORKER_DEFAULTS ], + dispatchers: [ DISPATCHER_DEFAULTS ], + scheduler: SCHEDULER_DEFAULTS + } end def recurring_tasks_config diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 3ac78d74..68a72d80 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -5,7 +5,7 @@ class Scheduler < Processes::Base include Processes::Runnable include LifecycleHooks - attr_reader :recurring_schedule + attr_reader :recurring_schedule, :polling_interval after_boot :run_start_hooks after_boot :schedule_recurring_tasks @@ -15,6 +15,8 @@ class Scheduler < Processes::Base def initialize(recurring_tasks:, **options) @recurring_schedule = RecurringSchedule.new(recurring_tasks) + options = options.dup.with_defaults(SolidQueue::Configuration::SCHEDULER_DEFAULTS) + @polling_interval = options[:polling_interval] super(**options) end @@ -24,8 +26,6 @@ def metadata end private - SLEEP_INTERVAL = 60 # Right now it doesn't matter, can be set to 1 in the future for dynamic tasks - def run loop do break if shutting_down? @@ -36,7 +36,7 @@ def run end end - interruptible_sleep(SLEEP_INTERVAL) + interruptible_sleep(polling_interval) end ensure SolidQueue.instrument(:shutdown_process, process: self) do diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 21f50f34..1e93681c 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -74,10 +74,7 @@ class SchedulerTest < ActiveSupport::TestCase end test "updates metadata after adding dynamic task post-start" do - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| - s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } - s.start - end + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) @@ -113,10 +110,7 @@ class SchedulerTest < ActiveSupport::TestCase arguments: [ 42 ] ) - scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}).tap do |s| - s.define_singleton_method(:interruptible_sleep) { |interval| sleep 0.1 } - s.start - end + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) From 421c1b31ac180e14348fa9469a36ad92dfd03267 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 19 Jun 2025 16:06:01 +0300 Subject: [PATCH 03/19] Fix abstraction for RecurringSchedule and Process --- lib/solid_queue/processes/registrable.rb | 4 ++++ lib/solid_queue/scheduler.rb | 7 ++---- .../scheduler/recurring_schedule.rb | 22 ++++++++++++++----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/solid_queue/processes/registrable.rb b/lib/solid_queue/processes/registrable.rb index c7428010..b53bd8fd 100644 --- a/lib/solid_queue/processes/registrable.rb +++ b/lib/solid_queue/processes/registrable.rb @@ -59,5 +59,9 @@ def heartbeat self.process = nil wake_up end + + def refresh_registered_process + process.update_columns(metadata: metadata.compact) + end end end diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index 68a72d80..f0464f2b 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -30,11 +30,8 @@ def run loop do break if shutting_down? - recurring_schedule.update_scheduled_tasks.tap do |updated_tasks| - if updated_tasks.any? - process.update_columns(metadata: metadata.compact) - end - end + recurring_schedule.reload! + refresh_registered_process if recurring_schedule.changed? interruptible_sleep(polling_interval) end diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index ea55f9f8..65f3be2d 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,12 +4,13 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :static_tasks, :configured_tasks, :scheduled_tasks + attr_reader :static_tasks, :configured_tasks, :scheduled_tasks, :changes def initialize(tasks) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) @configured_tasks = @static_tasks + dynamic_tasks @scheduled_tasks = Concurrent::Hash.new + @changes = Concurrent::Hash.new end def empty? @@ -44,10 +45,6 @@ def unschedule_old_dynamic_tasks end end - def update_scheduled_tasks - schedule_new_dynamic_tasks + unschedule_old_dynamic_tasks - end - def schedule_task(task) scheduled_tasks[task.key] = schedule(task) end @@ -65,6 +62,21 @@ def task_keys static_task_keys + dynamic_tasks.map(&:key) end + def reload! + { added_tasks: schedule_new_dynamic_tasks, + removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| + if values.any? + changes[key] = values + else + changes.delete(key) + end + end + end + + def changed? + @changes.any? + end + private def persist_static_tasks SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all From db9c061781b117fcaf18e927e71758d9cb56b48a Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 19 Jun 2025 17:35:14 +0300 Subject: [PATCH 04/19] Add create and destroy recurring task helpers --- lib/solid_queue.rb | 9 +++++++++ test/solid_queue_test.rb | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index e0d51c8c..0b8ea4e3 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -43,6 +43,15 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor + + def create_recurring_task(key, **attributes) + RecurringTask.create!(**attributes, key:, static: false) + end + + def destroy_recurring_task(id) + RecurringTask.dynamic.find(id).destroy! + end + [ Dispatcher, Scheduler, Worker ].each do |process| define_singleton_method(:"on_#{process.name.demodulize.downcase}_start") do |&block| process.on_start(&block) diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index d6d61b57..2c7bd00b 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -4,4 +4,31 @@ class SolidQueueTest < ActiveSupport::TestCase test "it has a version number" do assert SolidQueue::VERSION end + + test "creates recurring tasks" do + SolidQueue.create_recurring_task("test 1", command: "puts 1", schedule: "every hour") + SolidQueue.create_recurring_task("test 2", command: "puts 2", schedule: "every minute", static: true) + + assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) + assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) + end + + test "destroys recurring tasks" do + dynamic_task = SolidQueue::RecurringTask.create!( + key: "dynamic", command: "puts 'd'", schedule: "every day", static: false + ) + + static_task = SolidQueue::RecurringTask.create!( + key: "static", command: "puts 's'", schedule: "every week", static: true + ) + + SolidQueue.destroy_recurring_task(dynamic_task.id) + + assert_raises(ActiveRecord::RecordNotFound) do + SolidQueue.destroy_recurring_task(static_task.id) + end + + assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) + assert SolidQueue::RecurringTask.exists?(key: "static", static: true) + end end From b024ca631bdf1b58a51ba0c001e04105efe29a52 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Sat, 12 Jul 2025 12:28:18 +0300 Subject: [PATCH 05/19] Update README with Recurring tasks info --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7c13bda3..6329f377 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,17 @@ It is recommended to set this value less than or equal to the queue database's c - `concurrency_maintenance`: whether the dispatcher will perform the concurrency maintenance work. This is `true` by default, and it's useful if you don't use any [concurrency controls](#concurrency-controls) and want to disable it or if you run multiple dispatchers and want some of them to just dispatch jobs without doing anything else. +### Scheduler polling interval + +The scheduler process checks for due recurring tasks and reloads dynamic tasks at a configurable interval. You can set this interval using the `polling_interval` key under the `scheduler` section in your `config/queue.yml`: + +```yaml +scheduler: + polling_interval: 5 # seconds +``` + +This controls how frequently the scheduler wakes up to enqueue due recurring jobs and reload dynamic tasks. + ### Queue order and priorities As mentioned above, if you specify a list of queues for a worker, these will be polled in the order given, such as for the list `real_time,background`, no jobs will be taken from `background` unless there aren't any more jobs waiting in `real_time`. @@ -707,8 +718,6 @@ Rails.application.config.after_initialize do # or to_prepare end ``` -You can also dynamically add or remove recurring tasks by creating or deleting SolidQueue::RecurringTask records. It works the same way as with static tasks, except you must set the static field to false. Changes won’t be picked up immediately — they take effect after about a one-minute delay. - It's possible to run multiple schedulers with the same `recurring_tasks` configuration, for example, if you have multiple servers for redundancy, and you run the `scheduler` in more than one of them. To avoid enqueuing duplicate tasks at the same time, an entry in a new `solid_queue_recurring_executions` table is created in the same transaction as the job is enqueued. This table has a unique index on `task_key` and `run_at`, ensuring only one entry per task per time will be created. This only works if you have `preserve_finished_jobs` set to `true` (the default), and the guarantee applies as long as you keep the jobs around. **Note**: a single recurring schedule is supported, so you can have multiple schedulers using the same schedule, but not multiple schedulers using different configurations. @@ -734,6 +743,47 @@ my_periodic_resque_job: and the job will be enqueued via `perform_later` so it'll run in Resque. However, in this case we won't track any `solid_queue_recurring_execution` record for it and there won't be any guarantees that the job is enqueued only once each time. + +### Creating and Deleting Recurring Tasks Dynamically + +You can create and delete recurring tasks at runtime, without editing the configuration file. Use the following methods: + +#### Creating a recurring task + +```ruby +SolidQueue.schedule_recurring_task( + "my_dynamic_task", + command: "puts 'Hello from a dynamic task!'", + schedule: "every 10 minutes" +) +``` + +This will create a dynamic recurring task with the given key, command, and schedule. You can also use the `class` and `args` options as in the configuration file. + +#### Deleting a recurring task + +```ruby +SolidQueue.delete_recurring_task(task_id) +``` + +This will delete a dynamically scheduled recurring task by its ID. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. + +> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be deleted at runtime. Attempting to do so will raise an error. + +#### Example: Creating and deleting a recurring task + +```ruby +# Create a new dynamic recurring task +recurring_task = SolidQueue.schedule_recurring_task( + "cleanup_temp_files", + command: "TempFileCleaner.clean!", + schedule: "every day at 2am" +) + +# Delete the task later by ID +SolidQueue.delete_recurring_task(recurring_task.id) +``` + ## Inspiration Solid Queue has been inspired by [resque](https://github.com/resque/resque) and [GoodJob](https://github.com/bensheldon/good_job). We recommend checking out these projects as they're great examples from which we've learnt a lot. From 829a70670243bb3dea194cb842b12645f99742e9 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 21 Aug 2025 19:06:33 +0300 Subject: [PATCH 06/19] Use task key instead of id --- lib/solid_queue.rb | 4 ++-- test/solid_queue_test.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 0b8ea4e3..6d088de3 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -48,8 +48,8 @@ def create_recurring_task(key, **attributes) RecurringTask.create!(**attributes, key:, static: false) end - def destroy_recurring_task(id) - RecurringTask.dynamic.find(id).destroy! + def destroy_recurring_task(key) + RecurringTask.dynamic.find_by!(key:).destroy end [ Dispatcher, Scheduler, Worker ].each do |process| diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 2c7bd00b..316add46 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -22,10 +22,10 @@ class SolidQueueTest < ActiveSupport::TestCase key: "static", command: "puts 's'", schedule: "every week", static: true ) - SolidQueue.destroy_recurring_task(dynamic_task.id) + SolidQueue.destroy_recurring_task(dynamic_task.key) assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.destroy_recurring_task(static_task.id) + SolidQueue.destroy_recurring_task(static_task.key) end assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) From cd1dbe285ab03325e24884d85f68a56348792778 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 21 Aug 2025 19:07:03 +0300 Subject: [PATCH 07/19] Fix mismatches in readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6329f377..adaf419c 100644 --- a/README.md +++ b/README.md @@ -751,7 +751,7 @@ You can create and delete recurring tasks at runtime, without editing the config #### Creating a recurring task ```ruby -SolidQueue.schedule_recurring_task( +SolidQueue.create_recurring_task( "my_dynamic_task", command: "puts 'Hello from a dynamic task!'", schedule: "every 10 minutes" @@ -763,10 +763,10 @@ This will create a dynamic recurring task with the given key, command, and sched #### Deleting a recurring task ```ruby -SolidQueue.delete_recurring_task(task_id) +SolidQueue.destroy_recurring_task(key) ``` -This will delete a dynamically scheduled recurring task by its ID. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. +This will delete a dynamically scheduled recurring task by its key. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. > **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be deleted at runtime. Attempting to do so will raise an error. @@ -774,14 +774,14 @@ This will delete a dynamically scheduled recurring task by its ID. If you attemp ```ruby # Create a new dynamic recurring task -recurring_task = SolidQueue.schedule_recurring_task( +recurring_task = SolidQueue.create_recurring_task( "cleanup_temp_files", command: "TempFileCleaner.clean!", schedule: "every day at 2am" ) -# Delete the task later by ID -SolidQueue.delete_recurring_task(recurring_task.id) +# Delete the task later by key +SolidQueue.destroy_recurring_task("cleanup_temp_files") ``` ## Inspiration From 676a2a4b581b724021cea52ef75a36f79983f264 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:04:23 +0200 Subject: [PATCH 08/19] Use correct asserts in tests Two assertions were using assert value, message instead of assert_equal/assert_empty, meaning they always passed regardless of the actual metadata content. Fixed to use assert_empty. Also fixed a typo ("unschedule" -> "unscheduled"). --- test/unit/scheduler_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 1e93681c..aac9aa51 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -80,7 +80,7 @@ class SchedulerTest < ActiveSupport::TestCase process = SolidQueue::Process.first # initially there are no recurring_schedule keys - assert process.metadata, {} + assert_empty process.metadata # now create a dynamic task after the scheduler has booted SolidQueue::RecurringTask.create( @@ -124,8 +124,8 @@ class SchedulerTest < ActiveSupport::TestCase process.reload - # The task is unschedule after it's being removed, and it's reflected in the metadata - assert process.metadata, {} + # The task is unscheduled after it's been removed, and it's reflected in the metadata + assert_empty process.metadata ensure scheduler&.stop end From ae89324e310b04c16dce61144f1778775eea8d1a Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:04:45 +0200 Subject: [PATCH 09/19] Add missing platform to Gemfile.lock --- Gemfile.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile.lock b/Gemfile.lock index ecb7ccf8..1f613581 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -193,6 +193,7 @@ PLATFORMS arm64-darwin-22 arm64-darwin-23 arm64-darwin-24 + arm64-darwin-25 x86_64-darwin-21 x86_64-darwin-23 x86_64-linux From 5956d4e6235ee6c324bafef8533547b9e644367f Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:08:29 +0200 Subject: [PATCH 10/19] Move some Scheduler::RecurringSchedule methods to private --- .../scheduler/recurring_schedule.rb | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 65f3be2d..5b72d91a 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,7 +4,7 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :static_tasks, :configured_tasks, :scheduled_tasks, :changes + attr_reader :configured_tasks, :scheduled_tasks def initialize(tasks) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) @@ -28,23 +28,6 @@ def schedule_tasks end end - def dynamic_tasks - SolidQueue::RecurringTask.dynamic - end - - def schedule_new_dynamic_tasks - dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| - schedule_task(task) - end - end - - def unschedule_old_dynamic_tasks - (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| - scheduled_tasks[key].cancel - scheduled_tasks.delete(key) - end - end - def schedule_task(task) scheduled_tasks[task.key] = schedule(task) end @@ -54,10 +37,6 @@ def unschedule_tasks scheduled_tasks.clear end - def static_task_keys - static_tasks.map(&:key) - end - def task_keys static_task_keys + dynamic_tasks.map(&:key) end @@ -78,6 +57,29 @@ def changed? end private + attr_reader :static_tasks + + def dynamic_tasks + SolidQueue::RecurringTask.dynamic + end + + def static_task_keys + static_tasks.map(&:key) + end + + def schedule_new_dynamic_tasks + dynamic_tasks.where.not(key: scheduled_tasks.keys).each do |task| + schedule_task(task) + end + end + + def unschedule_old_dynamic_tasks + (scheduled_tasks.keys - SolidQueue::RecurringTask.pluck(:key)).each do |key| + scheduled_tasks[key].cancel + scheduled_tasks.delete(key) + end + end + def persist_static_tasks SolidQueue::RecurringTask.static.where.not(key: static_task_keys).delete_all SolidQueue::RecurringTask.create_or_update_all static_tasks From 8f0b5853dd1f5ab3e3054259e8e62d074d00df8c Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:09:26 +0200 Subject: [PATCH 11/19] Wrap reload in app executor DB queries in reload! (dynamic_tasks.where.not(...), RecurringTask.pluck(:key)) were not wrapped in the app executor, which could cause connection management issues. Wrapped in wrap_in_app_executor. --- lib/solid_queue/scheduler/recurring_schedule.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 5b72d91a..2dacada0 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -42,12 +42,14 @@ def task_keys end def reload! - { added_tasks: schedule_new_dynamic_tasks, - removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| - if values.any? - changes[key] = values - else - changes.delete(key) + wrap_in_app_executor do + { added_tasks: schedule_new_dynamic_tasks, + removed_tasks: unschedule_old_dynamic_tasks }.each do |key, values| + if values.any? + @changes[key] = values + else + @changes.delete(key) + end end end end From 31e0e59ae1b8ed0fbdb469385004345a21032b53 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:10:39 +0200 Subject: [PATCH 12/19] Fix empty? method in Scheduler::RecurringSchedule empty? checked stale configured_tasks (lib/solid_queue/scheduler/recurring_schedule.rb) -- configured_tasks was set once in initialize and never updated with dynamic tasks. This meant empty? could return true even when dynamic tasks existed, causing the scheduler to exit prematurely in inline mode. Changed empty? to check scheduled_tasks.empty? && dynamic_tasks.none?. --- lib/solid_queue/scheduler/recurring_schedule.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index 2dacada0..ff7e7a7b 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -14,7 +14,7 @@ def initialize(tasks) end def empty? - configured_tasks.empty? + scheduled_tasks.empty? && dynamic_tasks.none? end def schedule_tasks From bb1de62f308297337f003d806d7f1323f9a94264 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:13:21 +0200 Subject: [PATCH 13/19] Prevent stale change by adding clear_changes in scheduler's loop Changes not cleared after consumption (lib/solid_queue/scheduler.rb, lib/solid_queue/scheduler/recurring_schedule.rb) -- Added clear_changes method and call it in the scheduler's run loop after refresh_registered_process, preventing stale change state from persisting. --- lib/solid_queue/scheduler.rb | 6 +++++- lib/solid_queue/scheduler/recurring_schedule.rb | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/solid_queue/scheduler.rb b/lib/solid_queue/scheduler.rb index f0464f2b..920c5d3a 100644 --- a/lib/solid_queue/scheduler.rb +++ b/lib/solid_queue/scheduler.rb @@ -31,7 +31,11 @@ def run break if shutting_down? recurring_schedule.reload! - refresh_registered_process if recurring_schedule.changed? + + if recurring_schedule.changed? + refresh_registered_process + recurring_schedule.clear_changes + end interruptible_sleep(polling_interval) end diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index ff7e7a7b..dfe192f7 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -58,6 +58,10 @@ def changed? @changes.any? end + def clear_changes + @changes.clear + end + private attr_reader :static_tasks From 1cd5e3f0d575bc27a69a301304faffc201a92f2e Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:13:43 +0200 Subject: [PATCH 14/19] Add test for enqueuing the job --- test/unit/scheduler_test.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index aac9aa51..669afdcf 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -73,6 +73,22 @@ class SchedulerTest < ActiveSupport::TestCase end end + test "dynamic task actually enqueues jobs" do + SolidQueue::RecurringTask.create!( + key: "dynamic_enqueue_task", static: false, class_name: "AddToBufferJob", schedule: "every second", arguments: [ 42 ] + ) + + scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) + + wait_for_registered_processes(1, timeout: 1.second) + sleep 2 + + assert SolidQueue::Job.count >= 1, "Expected at least one job to be enqueued by the dynamic task" + assert_equal SolidQueue::Job.count, SolidQueue::RecurringExecution.count + ensure + scheduler&.stop + end + test "updates metadata after adding dynamic task post-start" do scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) From 463fdedf8fed977d83c5a2def520e885c7465702 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:18:54 +0200 Subject: [PATCH 15/19] Improve create_recurring_task - use RecurringTask .from_configuration --- README.md | 9 +++++---- lib/solid_queue.rb | 7 +++++-- test/solid_queue_test.rb | 9 +++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index adaf419c..ed00d7dc 100644 --- a/README.md +++ b/README.md @@ -753,12 +753,13 @@ You can create and delete recurring tasks at runtime, without editing the config ```ruby SolidQueue.create_recurring_task( "my_dynamic_task", - command: "puts 'Hello from a dynamic task!'", + class: "MyJob", + args: [1, 2], schedule: "every 10 minutes" ) ``` -This will create a dynamic recurring task with the given key, command, and schedule. You can also use the `class` and `args` options as in the configuration file. +This will create a dynamic recurring task with the given key, class, and schedule. The API accepts the same options as the YAML configuration: `class`, `args`, `command`, `schedule`, `queue`, `priority`, and `description`. #### Deleting a recurring task @@ -774,9 +775,9 @@ This will delete a dynamically scheduled recurring task by its key. If you attem ```ruby # Create a new dynamic recurring task -recurring_task = SolidQueue.create_recurring_task( +SolidQueue.create_recurring_task( "cleanup_temp_files", - command: "TempFileCleaner.clean!", + class: "TempFileCleanerJob", schedule: "every day at 2am" ) diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 6d088de3..7ca1fd03 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -44,8 +44,11 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor - def create_recurring_task(key, **attributes) - RecurringTask.create!(**attributes, key:, static: false) + def create_recurring_task(key, **options) + RecurringTask.from_configuration(key, **options).tap do |task| + task.static = false + task.save! + end end def destroy_recurring_task(key) diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 316add46..01c41a19 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -13,6 +13,15 @@ class SolidQueueTest < ActiveSupport::TestCase assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) end + test "creates recurring tasks with class and args (same keys as YAML config)" do + SolidQueue.create_recurring_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") + + task = SolidQueue::RecurringTask.find_by!(key: "test 3") + assert_equal "AddToBufferJob", task.class_name + assert_equal [ 42 ], task.arguments + assert_equal false, task.static + end + test "destroys recurring tasks" do dynamic_task = SolidQueue::RecurringTask.create!( key: "dynamic", command: "puts 'd'", schedule: "every day", static: false From 39749852809e625267cad4fae1cb4043c7a722a0 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Thu, 12 Feb 2026 16:30:06 +0200 Subject: [PATCH 16/19] Fix names for public methods --- README.md | 26 +++++++++++++------------- lib/solid_queue.rb | 4 ++-- test/solid_queue_test.rb | 16 ++++++++-------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ed00d7dc..63019c65 100644 --- a/README.md +++ b/README.md @@ -744,14 +744,14 @@ my_periodic_resque_job: and the job will be enqueued via `perform_later` so it'll run in Resque. However, in this case we won't track any `solid_queue_recurring_execution` record for it and there won't be any guarantees that the job is enqueued only once each time. -### Creating and Deleting Recurring Tasks Dynamically +### Scheduling and Unscheduling Recurring Tasks Dynamically -You can create and delete recurring tasks at runtime, without editing the configuration file. Use the following methods: +You can schedule and unschedule recurring tasks at runtime, without editing the configuration file. Use the following methods: -#### Creating a recurring task +#### Scheduling a recurring task ```ruby -SolidQueue.create_recurring_task( +SolidQueue.schedule_task( "my_dynamic_task", class: "MyJob", args: [1, 2], @@ -761,28 +761,28 @@ SolidQueue.create_recurring_task( This will create a dynamic recurring task with the given key, class, and schedule. The API accepts the same options as the YAML configuration: `class`, `args`, `command`, `schedule`, `queue`, `priority`, and `description`. -#### Deleting a recurring task +#### Unscheduling a recurring task ```ruby -SolidQueue.destroy_recurring_task(key) +SolidQueue.unschedule_task(key) ``` -This will delete a dynamically scheduled recurring task by its key. If you attempt to delete a static (configuration-defined) recurring task, an error will be raised. +This will delete a dynamically scheduled recurring task by its key. If you attempt to unschedule a static (configuration-defined) recurring task, an error will be raised. -> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be deleted at runtime. Attempting to do so will raise an error. +> **Note:** Static recurring tasks (those defined in `config/recurring.yml`) cannot be unscheduled at runtime. Attempting to do so will raise an error. -#### Example: Creating and deleting a recurring task +#### Example: Scheduling and unscheduling a recurring task ```ruby -# Create a new dynamic recurring task -SolidQueue.create_recurring_task( +# Schedule a new dynamic recurring task +SolidQueue.schedule_task( "cleanup_temp_files", class: "TempFileCleanerJob", schedule: "every day at 2am" ) -# Delete the task later by key -SolidQueue.destroy_recurring_task("cleanup_temp_files") +# Unschedule the task later by key +SolidQueue.unschedule_task("cleanup_temp_files") ``` ## Inspiration diff --git a/lib/solid_queue.rb b/lib/solid_queue.rb index 7ca1fd03..11994b76 100644 --- a/lib/solid_queue.rb +++ b/lib/solid_queue.rb @@ -44,14 +44,14 @@ module SolidQueue delegate :on_start, :on_stop, :on_exit, to: Supervisor - def create_recurring_task(key, **options) + def schedule_task(key, **options) RecurringTask.from_configuration(key, **options).tap do |task| task.static = false task.save! end end - def destroy_recurring_task(key) + def unschedule_task(key) RecurringTask.dynamic.find_by!(key:).destroy end diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 01c41a19..09140654 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -5,16 +5,16 @@ class SolidQueueTest < ActiveSupport::TestCase assert SolidQueue::VERSION end - test "creates recurring tasks" do - SolidQueue.create_recurring_task("test 1", command: "puts 1", schedule: "every hour") - SolidQueue.create_recurring_task("test 2", command: "puts 2", schedule: "every minute", static: true) + test "schedules recurring tasks" do + SolidQueue.schedule_task("test 1", command: "puts 1", schedule: "every hour") + SolidQueue.schedule_task("test 2", command: "puts 2", schedule: "every minute", static: true) assert SolidQueue::RecurringTask.exists?(key: "test 1", command: "puts 1", schedule: "every hour", static: false) assert SolidQueue::RecurringTask.exists?(key: "test 2", command: "puts 2", schedule: "every minute", static: false) end - test "creates recurring tasks with class and args (same keys as YAML config)" do - SolidQueue.create_recurring_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") + test "schedules recurring tasks with class and args (same keys as YAML config)" do + SolidQueue.schedule_task("test 3", class: "AddToBufferJob", args: [ 42 ], schedule: "every hour") task = SolidQueue::RecurringTask.find_by!(key: "test 3") assert_equal "AddToBufferJob", task.class_name @@ -22,7 +22,7 @@ class SolidQueueTest < ActiveSupport::TestCase assert_equal false, task.static end - test "destroys recurring tasks" do + test "unschedules recurring tasks" do dynamic_task = SolidQueue::RecurringTask.create!( key: "dynamic", command: "puts 'd'", schedule: "every day", static: false ) @@ -31,10 +31,10 @@ class SolidQueueTest < ActiveSupport::TestCase key: "static", command: "puts 's'", schedule: "every week", static: true ) - SolidQueue.destroy_recurring_task(dynamic_task.key) + SolidQueue.unschedule_task(dynamic_task.key) assert_raises(ActiveRecord::RecordNotFound) do - SolidQueue.destroy_recurring_task(static_task.key) + SolidQueue.unschedule_task(static_task.key) end assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) From 941ea869fad74e2c02015c3f168b4280cb26f765 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Fri, 13 Feb 2026 00:29:02 +0200 Subject: [PATCH 17/19] Prevent stale configured_tasks value --- lib/solid_queue/scheduler/recurring_schedule.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/solid_queue/scheduler/recurring_schedule.rb b/lib/solid_queue/scheduler/recurring_schedule.rb index dfe192f7..12c7af3d 100644 --- a/lib/solid_queue/scheduler/recurring_schedule.rb +++ b/lib/solid_queue/scheduler/recurring_schedule.rb @@ -4,15 +4,18 @@ module SolidQueue class Scheduler::RecurringSchedule include AppExecutor - attr_reader :configured_tasks, :scheduled_tasks + attr_reader :scheduled_tasks def initialize(tasks) @static_tasks = Array(tasks).map { |task| SolidQueue::RecurringTask.wrap(task) }.select(&:valid?) - @configured_tasks = @static_tasks + dynamic_tasks @scheduled_tasks = Concurrent::Hash.new @changes = Concurrent::Hash.new end + def configured_tasks + static_tasks + dynamic_tasks.to_a + end + def empty? scheduled_tasks.empty? && dynamic_tasks.none? end @@ -38,7 +41,7 @@ def unschedule_tasks end def task_keys - static_task_keys + dynamic_tasks.map(&:key) + static_task_keys + dynamic_tasks.pluck(:key) end def reload! From 29778e7970b91d61d0ec721339b5ebf1d1fc5543 Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Fri, 13 Feb 2026 00:29:40 +0200 Subject: [PATCH 18/19] Use wrap_in_app_executor for refresh_registered_process --- lib/solid_queue/processes/registrable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/solid_queue/processes/registrable.rb b/lib/solid_queue/processes/registrable.rb index b53bd8fd..4edf03e3 100644 --- a/lib/solid_queue/processes/registrable.rb +++ b/lib/solid_queue/processes/registrable.rb @@ -61,7 +61,7 @@ def heartbeat end def refresh_registered_process - process.update_columns(metadata: metadata.compact) + wrap_in_app_executor { process&.update_columns(metadata: metadata.compact) } end end end From 62bfa3772e5530c47ea68f9ee8eaafdcadfdcf1b Mon Sep 17 00:00:00 2001 From: Vladyslav Davydenko Date: Fri, 13 Feb 2026 00:31:58 +0200 Subject: [PATCH 19/19] Update tests and README --- README.md | 2 + lib/solid_queue/configuration.rb | 3 ++ test/solid_queue_test.rb | 14 ++++++ test/unit/scheduler_test.rb | 74 ++++++++++++++++---------------- 4 files changed, 57 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 63019c65..536c2f86 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,8 @@ scheduler: This controls how frequently the scheduler wakes up to enqueue due recurring jobs and reload dynamic tasks. +> **Note:** The scheduler process always starts by default to support dynamic recurring tasks, even if no static tasks are configured in `config/recurring.yml`. If you don't use recurring tasks at all, you can disable the scheduler by setting `SOLID_QUEUE_SKIP_RECURRING=true` or passing `skip_recurring: true` in the configuration. + ### Queue order and priorities As mentioned above, if you specify a list of queues for a worker, these will be polled in the order given, such as for the list `real_time,background`, no jobs will be taken from `background` unless there aren't any more jobs waiting in `real_time`. diff --git a/lib/solid_queue/configuration.rb b/lib/solid_queue/configuration.rb index 244444bd..21f7ad5d 100644 --- a/lib/solid_queue/configuration.rb +++ b/lib/solid_queue/configuration.rb @@ -143,6 +143,9 @@ def dispatchers def schedulers return [] if skip_recurring_tasks? + # Always start a scheduler (even with no static recurring tasks) to support + # dynamic tasks that may be added at runtime via SolidQueue.schedule_task. + # Use skip_recurring: true or SOLID_QUEUE_SKIP_RECURRING=true to disable. [ Process.new(:scheduler, { recurring_tasks:, **scheduler_options.with_defaults(SCHEDULER_DEFAULTS) }) ] end diff --git a/test/solid_queue_test.rb b/test/solid_queue_test.rb index 09140654..542c3d82 100644 --- a/test/solid_queue_test.rb +++ b/test/solid_queue_test.rb @@ -40,4 +40,18 @@ class SolidQueueTest < ActiveSupport::TestCase assert_not SolidQueue::RecurringTask.exists?(key: "dynamic", static: false) assert SolidQueue::RecurringTask.exists?(key: "static", static: true) end + + test "schedule_task with duplicate key raises error" do + SolidQueue.schedule_task("duplicate_test", command: "puts 1", schedule: "every hour") + + assert_raises(ActiveRecord::RecordNotUnique) do + SolidQueue.schedule_task("duplicate_test", command: "puts 2", schedule: "every minute") + end + end + + test "unschedule_task with nonexistent key raises RecordNotFound" do + assert_raises(ActiveRecord::RecordNotFound) do + SolidQueue.unschedule_task("nonexistent_key") + end + end end diff --git a/test/unit/scheduler_test.rb b/test/unit/scheduler_test.rb index 669afdcf..882dcd72 100644 --- a/test/unit/scheduler_test.rb +++ b/test/unit/scheduler_test.rb @@ -81,10 +81,12 @@ class SchedulerTest < ActiveSupport::TestCase scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) - sleep 2 + wait_while_with_timeout(3.seconds) { SolidQueue::Job.count < 1 } - assert SolidQueue::Job.count >= 1, "Expected at least one job to be enqueued by the dynamic task" - assert_equal SolidQueue::Job.count, SolidQueue::RecurringExecution.count + skip_active_record_query_cache do + assert SolidQueue::Job.count >= 1, "Expected at least one job to be enqueued by the dynamic task" + assert_equal SolidQueue::Job.count, SolidQueue::RecurringExecution.count + end ensure scheduler&.stop end @@ -94,54 +96,54 @@ class SchedulerTest < ActiveSupport::TestCase wait_for_registered_processes(1, timeout: 1.second) - process = SolidQueue::Process.first - # initially there are no recurring_schedule keys - assert_empty process.metadata - - # now create a dynamic task after the scheduler has booted - SolidQueue::RecurringTask.create( - key: "new_dynamic_task", - static: false, - class_name: "AddToBufferJob", - schedule: "every second", - arguments: [ 42 ] - ) - - sleep 1 - - process.reload - - # metadata should now include the new key - assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] + skip_active_record_query_cache do + process = SolidQueue::Process.first + # initially there are no recurring_schedule keys + assert_empty process.metadata + + # now create a dynamic task after the scheduler has booted + SolidQueue::RecurringTask.create!( + key: "new_dynamic_task", + static: false, + class_name: "AddToBufferJob", + schedule: "every second", + arguments: [ 42 ] + ) + + wait_while_with_timeout(3.seconds) { process.reload.metadata.empty? } + + # metadata should now include the new key + assert_metadata process, recurring_schedule: [ "new_dynamic_task" ] + end ensure scheduler&.stop end test "updates metadata after removing dynamic task post-start" do - old_dynamic_task = SolidQueue::RecurringTask.create( - key: "old_dynamic_task", - static: false, + old_dynamic_task = SolidQueue::RecurringTask.create!( + key: "old_dynamic_task", + static: false, class_name: "AddToBufferJob", - schedule: "every second", - arguments: [ 42 ] + schedule: "every second", + arguments: [ 42 ] ) scheduler = SolidQueue::Scheduler.new(recurring_tasks: {}, polling_interval: 0.1).tap(&:start) wait_for_registered_processes(1, timeout: 1.second) - process = SolidQueue::Process.first - # initially there is one recurring_schedule key - assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] - - old_dynamic_task.destroy + skip_active_record_query_cache do + process = SolidQueue::Process.first + # initially there is one recurring_schedule key + assert_metadata process, recurring_schedule: [ "old_dynamic_task" ] - sleep 1 + old_dynamic_task.destroy - process.reload + wait_while_with_timeout(3.seconds) { process.reload.metadata.present? } - # The task is unscheduled after it's been removed, and it's reflected in the metadata - assert_empty process.metadata + # The task is unscheduled after it's been removed, and it's reflected in the metadata + assert_empty process.metadata + end ensure scheduler&.stop end