Add solid-queue

.dockerignore

@@ -1,5 +1,30 @@
-log/*
-tmp/*
 Dockerfile
-.env*
 .git
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore all logfiles and tempfiles.
+/log/*
+/tmp/*
+!/log/.keep
+!/tmp/.keep
+
+# Ignore pidfiles, but keep the directory.
+/tmp/pids/*
+!/tmp/pids/
+!/tmp/pids/.keep
+
+# Ignore uploaded files in development.
+/storage/*
+!/storage/.keep
+/tmp/storage/*
+!/tmp/storage/
+!/tmp/storage/.keep
+
+/public/assets
+
+# Ignore master key for decrypting credentials and more.
+/config/master.key
+
+/.env*.local

Gemfile

@@ -5,8 +5,10 @@ ruby "3.1.2"
 
 gem "bootsnap", require: false
 gem "clearance"
+gem "combine_pdf"
 gem "dotenv-rails"
 gem "importmap-rails"
+gem "nokogiri"
 gem "paper_trail"
 gem "pg", "~> 1.1"
 gem "propshaft"
@@ -14,9 +16,8 @@ gem "puma", "~> 6.0"
 gem "pundit"
 gem "rails", "~> 7.1"
 gem "ransack"
-gem "nokogiri"
 gem "rubyzip"
-gem "combine_pdf"
+gem "solid_queue"
 
 group :development, :test do
   gem "debug", platforms: %i[ mri mingw x64_mingw ]

Gemfile.lock

@@ -111,10 +111,15 @@ GEM
     email_validator (2.2.4)
       activemodel
     erubi (1.12.0)
+    et-orbi (1.2.11)
+      tzinfo
     ffi (1.16.3)
     ffi-compiler (1.3.2)
       ffi (>= 1.15.5)
       rake
+    fugit (1.9.0)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
     i18n (1.14.4)
@@ -168,6 +173,7 @@ GEM
       nio4r (~> 2.0)
     pundit (2.3.1)
       activesupport (>= 3.0.0)
+    raabro (1.4.0)
     racc (1.7.3)
     rack (3.0.10)
     rack-session (2.0.0)
@@ -219,6 +225,12 @@ GEM
       rack (>= 1.4)
     ruby-rc4 (0.1.5)
     rubyzip (2.3.2)
+    solid_queue (0.3.0)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (~> 1.2.2)
+      fugit (~> 1.9.0)
+      railties (>= 7.1)
     stringio (3.1.0)
     thor (1.3.1)
     timecop (0.9.8)
@@ -255,6 +267,7 @@ DEPENDENCIES
   rails (~> 7.1)
   ransack
   rubyzip
+  solid_queue
   timecop
   web-console
 

app/jobs/notify_job.rb

@@ -0,0 +1,5 @@
+class NotifyJob < ApplicationJob
+  def perform
+    Notification.send_scheduled_for_today
+  end
+end

app/jobs/sync_job.rb

@@ -0,0 +1,5 @@
+class SyncJob < ApplicationJob
+  def perform
+    Member.reset_all_status!
+  end
+end

config/environments/production.rb

@@ -63,7 +63,7 @@ Rails.application.configure do
   # config.cache_store = :mem_cache_store
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  # config.active_job.queue_adapter = :resque
+  config.active_job.queue_adapter = :solid_queue
   # config.active_job.queue_name_prefix = "saucy_production"
 
   config.action_mailer.perform_caching = false

config/puma.rb

@@ -41,3 +41,4 @@ pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" }
 
 # Allow puma to be restarted by `bin/rails restart` command.
 plugin :tmp_restart
+plugin :solid_queue

config/solid_queue.yml

@@ -0,0 +1,25 @@
+default: &default
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+      recurring_tasks:
+        sync:
+          class: SyncJob
+          schedule: every 5 minutes
+        sync:
+          class: NotifyJob
+          schedule: every 2 hours
+  workers:
+    - queues: "*"
+      threads: 5
+      processes: 1
+      polling_interval: 0.1
+
+development:
+ <<: *default
+
+test:
+ <<: *default
+
+production:
+ <<: *default

db/migrate/20240328101456_create_solid_queue_tables.solid_queue.rb

@@ -0,0 +1,101 @@
+# This migration comes from solid_queue (originally 20231211200639)
+class CreateSolidQueueTables < ActiveRecord::Migration[7.0]
+  def change
+    create_table :solid_queue_jobs do |t|
+      t.string :queue_name, null: false
+      t.string :class_name, null: false, index: true
+      t.text :arguments
+      t.integer :priority, default: 0, null: false
+      t.string :active_job_id, index: true
+      t.datetime :scheduled_at
+      t.datetime :finished_at, index: true
+      t.string :concurrency_key
+
+      t.timestamps
+
+      t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering"
+      t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    create_table :solid_queue_scheduled_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.datetime :scheduled_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all"
+    end
+
+    create_table :solid_queue_ready_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all"
+      t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue"
+    end
+
+    create_table :solid_queue_claimed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.bigint :process_id
+      t.datetime :created_at, null: false
+
+      t.index [ :process_id, :job_id ]
+    end
+
+    create_table :solid_queue_blocked_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.string :concurrency_key, null: false
+      t.datetime :expires_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance"
+    end
+
+    create_table :solid_queue_failed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.text :error
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_pauses do |t|
+      t.string :queue_name, null: false, index: { unique: true }
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_processes do |t|
+      t.string :kind, null: false
+      t.datetime :last_heartbeat_at, null: false, index: true
+      t.bigint :supervisor_id, index: true
+
+      t.integer :pid, null: false
+      t.string :hostname
+      t.text :metadata
+
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_semaphores do |t|
+      t.string :key, null: false, index: { unique: true }
+      t.integer :value, default: 1, null: false
+      t.datetime :expires_at, null: false, index: true
+
+      t.timestamps
+
+      t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value"
+    end
+
+    add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+    add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end

db/migrate/20240328101457_add_missing_index_to_blocked_executions.solid_queue.rb

@@ -0,0 +1,6 @@
+# This migration comes from solid_queue (originally 20240110143450)
+class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1]
+  def change
+    add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ], name: "index_solid_queue_blocked_executions_for_release"
+  end
+end

db/migrate/20240328101458_create_recurring_executions.solid_queue.rb

@@ -0,0 +1,15 @@
+# This migration comes from solid_queue (originally 20240218110712)
+class CreateRecurringExecutions < ActiveRecord::Migration[7.1]
+  def change
+    create_table :solid_queue_recurring_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :task_key, null: false
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false
+
+      t.index [ :task_key, :run_at ], unique: true
+    end
+
+    add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2024_03_28_095519) do
+ActiveRecord::Schema[7.1].define(version: 2024_03_28_101458) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
@@ -68,6 +68,109 @@ ActiveRecord::Schema[7.1].define(version: 2024_03_28_095519) do
     t.index ["member_id"], name: "index_payments_on_member_id"
   end
 
+  create_table "solid_queue_blocked_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.string "queue_name", null: false
+    t.integer "priority", default: 0, null: false
+    t.string "concurrency_key", null: false
+    t.datetime "expires_at", null: false
+    t.datetime "created_at", null: false
+    t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release"
+    t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance"
+    t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true
+  end
+
+  create_table "solid_queue_claimed_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.bigint "process_id"
+    t.datetime "created_at", null: false
+    t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true
+    t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id"
+  end
+
+  create_table "solid_queue_failed_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.text "error"
+    t.datetime "created_at", null: false
+    t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true
+  end
+
+  create_table "solid_queue_jobs", force: :cascade do |t|
+    t.string "queue_name", null: false
+    t.string "class_name", null: false
+    t.text "arguments"
+    t.integer "priority", default: 0, null: false
+    t.string "active_job_id"
+    t.datetime "scheduled_at"
+    t.datetime "finished_at"
+    t.string "concurrency_key"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id"
+    t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name"
+    t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at"
+    t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering"
+    t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting"
+  end
+
+  create_table "solid_queue_pauses", force: :cascade do |t|
+    t.string "queue_name", null: false
+    t.datetime "created_at", null: false
+    t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true
+  end
+
+  create_table "solid_queue_processes", force: :cascade do |t|
+    t.string "kind", null: false
+    t.datetime "last_heartbeat_at", null: false
+    t.bigint "supervisor_id"
+    t.integer "pid", null: false
+    t.string "hostname"
+    t.text "metadata"
+    t.datetime "created_at", null: false
+    t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at"
+    t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id"
+  end
+
+  create_table "solid_queue_ready_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.string "queue_name", null: false
+    t.integer "priority", default: 0, null: false
+    t.datetime "created_at", null: false
+    t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true
+    t.index ["priority", "job_id"], name: "index_solid_queue_poll_all"
+    t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue"
+  end
+
+  create_table "solid_queue_recurring_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.string "task_key", null: false
+    t.datetime "run_at", null: false
+    t.datetime "created_at", null: false
+    t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true
+    t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true
+  end
+
+  create_table "solid_queue_scheduled_executions", force: :cascade do |t|
+    t.bigint "job_id", null: false
+    t.string "queue_name", null: false
+    t.integer "priority", default: 0, null: false
+    t.datetime "scheduled_at", null: false
+    t.datetime "created_at", null: false
+    t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true
+    t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all"
+  end
+
+  create_table "solid_queue_semaphores", force: :cascade do |t|
+    t.string "key", null: false
+    t.integer "value", default: 1, null: false
+    t.datetime "expires_at", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at"
+    t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value"
+    t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true
+  end
+
   create_table "users", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
@@ -94,4 +197,10 @@ ActiveRecord::Schema[7.1].define(version: 2024_03_28_095519) do
   add_foreign_key "contributions", "members"
   add_foreign_key "notifications", "members"
   add_foreign_key "payments", "members"
+  add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
+  add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade
 end

lib/tasks/saucy.rake

@@ -2,12 +2,12 @@ desc "Application specific tasks"
 namespace :saucy do
   desc "Background sync operations"
   task sync: :environment do
-    Member.reset_all_status!
+    SyncJob.perform_now
   end
 
   desc "Send daily email notifications"
   task notify: :environment do
-    Notification.send_scheduled_for_today
+    NotifyJob.perform_now
   end
 
   desc "Import data from CiviCRM XML files"