diff --git a/.dockerignore b/.dockerignore index 0769f73..0566cf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,30 @@ -log/* -tmp/* Dockerfile -.env* .git + +# Ignore bundler config. +/.bundle + +# Ignore all logfiles and tempfiles. +/log/* +/tmp/* +!/log/.keep +!/tmp/.keep + +# Ignore pidfiles, but keep the directory. +/tmp/pids/* +!/tmp/pids/ +!/tmp/pids/.keep + +# Ignore uploaded files in development. +/storage/* +!/storage/.keep +/tmp/storage/* +!/tmp/storage/ +!/tmp/storage/.keep + +/public/assets + +# Ignore master key for decrypting credentials and more. +/config/master.key + +/.env*.local diff --git a/Gemfile b/Gemfile index 99a3aed..eb4f6ba 100644 --- a/Gemfile +++ b/Gemfile @@ -5,8 +5,10 @@ ruby "3.1.2" gem "bootsnap", require: false gem "clearance" +gem "combine_pdf" gem "dotenv-rails" gem "importmap-rails" +gem "nokogiri" gem "paper_trail" gem "pg", "~> 1.1" gem "propshaft" @@ -14,9 +16,8 @@ gem "puma", "~> 6.0" gem "pundit" gem "rails", "~> 7.1" gem "ransack" -gem "nokogiri" gem "rubyzip" -gem "combine_pdf" +gem "solid_queue" group :development, :test do gem "debug", platforms: %i[ mri mingw x64_mingw ] diff --git a/Gemfile.lock b/Gemfile.lock index c3dd221..29a9a9c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -111,10 +111,15 @@ GEM email_validator (2.2.4) activemodel erubi (1.12.0) + et-orbi (1.2.11) + tzinfo ffi (1.16.3) ffi-compiler (1.3.2) ffi (>= 1.15.5) rake + fugit (1.9.0) + et-orbi (~> 1, >= 1.2.7) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) i18n (1.14.4) @@ -168,6 +173,7 @@ GEM nio4r (~> 2.0) pundit (2.3.1) activesupport (>= 3.0.0) + raabro (1.4.0) racc (1.7.3) rack (3.0.10) rack-session (2.0.0) @@ -219,6 +225,12 @@ GEM rack (>= 1.4) ruby-rc4 (0.1.5) rubyzip (2.3.2) + solid_queue (0.3.0) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (~> 1.2.2) + fugit (~> 1.9.0) + railties (>= 7.1) stringio (3.1.0) thor (1.3.1) timecop (0.9.8) @@ -255,6 +267,7 @@ DEPENDENCIES rails (~> 7.1) ransack rubyzip + solid_queue timecop web-console diff --git a/app/jobs/notify_job.rb b/app/jobs/notify_job.rb new file mode 100644 index 0000000..3a7ca5b --- /dev/null +++ b/app/jobs/notify_job.rb @@ -0,0 +1,5 @@ +class NotifyJob < ApplicationJob + def perform + Notification.send_scheduled_for_today + end +end diff --git a/app/jobs/sync_job.rb b/app/jobs/sync_job.rb new file mode 100644 index 0000000..ccdb272 --- /dev/null +++ b/app/jobs/sync_job.rb @@ -0,0 +1,5 @@ +class SyncJob < ApplicationJob + def perform + Member.reset_all_status! + end +end diff --git a/config/environments/production.rb b/config/environments/production.rb index 8d489e1..0b6c873 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -63,7 +63,7 @@ Rails.application.configure do # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue # config.active_job.queue_name_prefix = "saucy_production" config.action_mailer.perform_caching = false diff --git a/config/puma.rb b/config/puma.rb index daaf036..d4f1ae1 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -41,3 +41,4 @@ pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart +plugin :solid_queue diff --git a/config/solid_queue.yml b/config/solid_queue.yml new file mode 100644 index 0000000..9aadbbb --- /dev/null +++ b/config/solid_queue.yml @@ -0,0 +1,25 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + recurring_tasks: + sync: + class: SyncJob + schedule: every 5 minutes + sync: + class: NotifyJob + schedule: every 2 hours + workers: + - queues: "*" + threads: 5 + processes: 1 + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/db/migrate/20240328101456_create_solid_queue_tables.solid_queue.rb b/db/migrate/20240328101456_create_solid_queue_tables.solid_queue.rb new file mode 100644 index 0000000..6d8c4f6 --- /dev/null +++ b/db/migrate/20240328101456_create_solid_queue_tables.solid_queue.rb @@ -0,0 +1,101 @@ +# This migration comes from solid_queue (originally 20231211200639) +class CreateSolidQueueTables < ActiveRecord::Migration[7.0] + def change + create_table :solid_queue_jobs do |t| + t.string :queue_name, null: false + t.string :class_name, null: false, index: true + t.text :arguments + t.integer :priority, default: 0, null: false + t.string :active_job_id, index: true + t.datetime :scheduled_at + t.datetime :finished_at, index: true + t.string :concurrency_key + + t.timestamps + + t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering" + t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table :solid_queue_scheduled_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.datetime :scheduled_at, null: false + + t.datetime :created_at, null: false + + t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all" + end + + create_table :solid_queue_ready_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + + t.datetime :created_at, null: false + + t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all" + t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue" + end + + create_table :solid_queue_claimed_executions do |t| + t.references :job, index: { unique: true }, null: false + t.bigint :process_id + t.datetime :created_at, null: false + + t.index [ :process_id, :job_id ] + end + + create_table :solid_queue_blocked_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.string :concurrency_key, null: false + t.datetime :expires_at, null: false + + t.datetime :created_at, null: false + + t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance" + end + + create_table :solid_queue_failed_executions do |t| + t.references :job, index: { unique: true }, null: false + t.text :error + t.datetime :created_at, null: false + end + + create_table :solid_queue_pauses do |t| + t.string :queue_name, null: false, index: { unique: true } + t.datetime :created_at, null: false + end + + create_table :solid_queue_processes do |t| + t.string :kind, null: false + t.datetime :last_heartbeat_at, null: false, index: true + t.bigint :supervisor_id, index: true + + t.integer :pid, null: false + t.string :hostname + t.text :metadata + + t.datetime :created_at, null: false + end + + create_table :solid_queue_semaphores do |t| + t.string :key, null: false, index: { unique: true } + t.integer :value, default: 1, null: false + t.datetime :expires_at, null: false, index: true + + t.timestamps + + t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value" + end + + add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/migrate/20240328101457_add_missing_index_to_blocked_executions.solid_queue.rb b/db/migrate/20240328101457_add_missing_index_to_blocked_executions.solid_queue.rb new file mode 100644 index 0000000..2d96b10 --- /dev/null +++ b/db/migrate/20240328101457_add_missing_index_to_blocked_executions.solid_queue.rb @@ -0,0 +1,6 @@ +# This migration comes from solid_queue (originally 20240110143450) +class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1] + def change + add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ], name: "index_solid_queue_blocked_executions_for_release" + end +end diff --git a/db/migrate/20240328101458_create_recurring_executions.solid_queue.rb b/db/migrate/20240328101458_create_recurring_executions.solid_queue.rb new file mode 100644 index 0000000..4c49821 --- /dev/null +++ b/db/migrate/20240328101458_create_recurring_executions.solid_queue.rb @@ -0,0 +1,15 @@ +# This migration comes from solid_queue (originally 20240218110712) +class CreateRecurringExecutions < ActiveRecord::Migration[7.1] + def change + create_table :solid_queue_recurring_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :task_key, null: false + t.datetime :run_at, null: false + t.datetime :created_at, null: false + + t.index [ :task_key, :run_at ], unique: true + end + + add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index c9200a4..eb7c8f8 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_03_28_095519) do +ActiveRecord::Schema[7.1].define(version: 2024_03_28_101458) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -68,6 +68,109 @@ ActiveRecord::Schema[7.1].define(version: 2024_03_28_095519) do t.index ["member_id"], name: "index_payments_on_member_id" end + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + create_table "users", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -94,4 +197,10 @@ ActiveRecord::Schema[7.1].define(version: 2024_03_28_095519) do add_foreign_key "contributions", "members" add_foreign_key "notifications", "members" add_foreign_key "payments", "members" + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade end diff --git a/lib/tasks/saucy.rake b/lib/tasks/saucy.rake index de84d94..ad552cf 100644 --- a/lib/tasks/saucy.rake +++ b/lib/tasks/saucy.rake @@ -2,12 +2,12 @@ desc "Application specific tasks" namespace :saucy do desc "Background sync operations" task sync: :environment do - Member.reset_all_status! + SyncJob.perform_now end desc "Send daily email notifications" task notify: :environment do - Notification.send_scheduled_for_today + NotifyJob.perform_now end desc "Import data from CiviCRM XML files"