From ca8c1cf2b5416924c4379d43d5c943928377747d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 23 Nov 2024 14:43:04 +0200 Subject: [PATCH] Add support for Valkey and default to using it instead of KeyDB Hopefully fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3544 --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++ group_vars/matrix_servers | 51 +++++++++++++++++++++++++++++++++------ requirements.yml | 3 +++ setup.yml | 2 ++ 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94ad55ef9..cb71f39de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,45 @@ +# 2024-11-23 + +## (Backward Compatibility Break) The playbook now defaults to Valkey, instead of KeyDB + +**TLDR**: if the playbook installed KeyDB (or Redis) as a dependency for you before, it will now replace it with [Valkey](https://valkey.io/) (a drop-in alternative). We [previously switched from Redis to KeyDB](#backward-compatibility-break-the-playbook-now-defaults-to-keydb-instead-of-redis), but Valkey is a better alternative, so we're switching again. + +The playbook used to install Redis or KeyDB if services have a need for a Redis-compatible implementation ([enabling worker support for Synapse](docs/configuring-playbook-synapse.md#load-balancing-with-workers), [enabling Hookshot encryption](docs/configuring-playbook-bridge-hookshot.md#end-to-bridge-encryption), etc.). + +Earlier this year, we switched from Redis to KeyDB - see [(Backward Compatibility Break) The playbook now defaults to KeyDB, instead of Redis](#backward-compatibility-break-the-playbook-now-defaults-to-keydb-instead-of-redis). + +Because Valkey seems to be a better successor to Redis (than KeyDB) and likely doesn't suffer from [issues like this one](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/3544), we now replace KeyDB with Valkey. + +Valkey (like KeyDB and Redis in the past) is an implicitly enabled dependency - you don't need custom configuration in `vars.yml` to enable it. + +Next time your run the playbook (via the `setup-all` tag), **KeyDB will be automatically uninstalled and replaced with Valkey**. Some Synapse downtime may occur while the switch happens. + +Users on `arm32` should be aware that there's **neither a prebuilt `arm32` container image for Valkey**, nor the Valkey role supports self-building yet. Users on this architecture likely don't run Synapse with workers, etc., so they're likely in no need of Valkey (or Redis/KeyDB). If Redis is necessary in an `arm32` deployment, disabling Valkey and making the playbook fall back to Redis is possible (see below). + +**The playbook still supports Redis** and you can keep using Redis (for now) if you'd like, by adding this additional configuration to your `vars.yml` file: + +```yml +# Explicitly disable both Valkey and KeyDB. +# +# Redis will be auto-enabled if necessary, +# because there's no other Redis-compatible implementation being enabled. +valkey_enabled: false +keydb_enabled: false +``` + +**The playbook still supports KeyDB** and you can keep using KeyDB (for now) if you'd like, by adding this additional configuration to your `vars.yml` file: + +```yml +# Explicitly disable Valkey enable KeyDB. +# +# Redis will not be auto-enabled beandcause a Redis-compatible implementation (KeyDB) is enabled. +valkey_enabled: false +keydb_enabled: true +``` + +At some point in time in the future, we'll remove both KeyDB and Redis from the playbook, so we recommend that you migrate to Valkey earlier anyway. + + # 2024-11-14 ## HTTP-compression support for Traefik-based setups diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index e21fdc749..86572a0f9 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -436,6 +436,8 @@ devture_systemd_service_manager_services_list_auto: | + ([{'name': (keydb_identifier + '.service'), 'priority': 750, 'groups': ['matrix', 'keydb']}] if keydb_enabled else []) + + ([{'name': (valkey_identifier + '.service'), 'priority': 750, 'groups': ['matrix', 'valkey']}] if valkey_enabled else []) + + ([{'name': 'matrix-pantalaimon.service', 'priority': 4000, 'groups': ['matrix', 'pantalaimon']}] if matrix_pantalaimon_enabled else []) + ([{'name': 'matrix-registration.service', 'priority': 4000, 'groups': ['matrix', 'registration', 'matrix-registration']}] if matrix_registration_enabled else []) @@ -2203,12 +2205,14 @@ matrix_hookshot_systemd_wanted_services_list: | ([(redis_identifier + '.service')] if redis_enabled and matrix_hookshot_cache_redis_host == redis_identifier else []) + ([(keydb_identifier + '.service')] if keydb_enabled and matrix_hookshot_cache_redis_host == keydb_identifier else []) + + + ([(valkey_identifier + '.service')] if valkey_enabled and matrix_hookshot_cache_redis_host == valkey_identifier else []) }} # Hookshot's experimental encryption feature (and possibly others) may benefit from Redis, if available. # We only connect to Redis if encryption is enabled (not for everyone who has Redis enabled), # because connectivity is still potentially troublesome and is to be investigated. -matrix_hookshot_cache_redis_host: "{{ redis_identifier if redis_enabled and matrix_hookshot_experimental_encryption_enabled else (keydb_identifier if keydb_enabled and matrix_hookshot_experimental_encryption_enabled else '') }}" +matrix_hookshot_cache_redis_host: "{{ valkey_identifier if valkey_enabled else (redis_identifier if redis_enabled else (keydb_identifier if keydb_enabled else '')) }}" matrix_hookshot_container_network: "{{ matrix_addons_container_network }}" @@ -2221,6 +2225,8 @@ matrix_hookshot_container_additional_networks_auto: | + ([keydb_container_network] if keydb_enabled and matrix_hookshot_cache_redis_host == keydb_identifier else []) + + ([valkey_container_network] if valkey_enabled and matrix_hookshot_cache_redis_host == valkey_identifier else []) + + ([matrix_playbook_reverse_proxyable_services_additional_network] if matrix_playbook_reverse_proxyable_services_additional_network and matrix_hookshot_container_labels_traefik_enabled else []) ) | unique }} @@ -4393,11 +4399,11 @@ ntfy_visitor_request_limit_exempt_hosts_hostnames_auto: | ###################################################################### # -# etke/redis +# redis # ###################################################################### -redis_enabled: "{{ not keydb_enabled and (matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled)) }}" +redis_enabled: "{{ not (keydb_enabled or valkey_enabled) and (matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled)) }}" redis_identifier: matrix-redis @@ -4408,7 +4414,7 @@ redis_base_path: "{{ matrix_base_data_path }}/redis" ###################################################################### # -# /etke/redis +# /redis # ###################################################################### @@ -4418,7 +4424,7 @@ redis_base_path: "{{ matrix_base_data_path }}/redis" # ###################################################################### -keydb_enabled: "{{ matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled) }}" +keydb_enabled: false keydb_identifier: matrix-keydb @@ -4442,6 +4448,31 @@ keydb_arch: |- # ###################################################################### + +###################################################################### +# +# valkey +# +###################################################################### + +valkey_enabled: "{{ matrix_synapse_workers_enabled or (matrix_hookshot_enabled and matrix_hookshot_experimental_encryption_enabled) }}" + +valkey_identifier: matrix-valkey + +valkey_uid: "{{ matrix_user_uid }}" +valkey_gid: "{{ matrix_user_gid }}" + +valkey_base_path: "{{ matrix_base_data_path }}/valkey" + +valkey_arch: "{{ matrix_architecture }}" + +###################################################################### +# +# valkey +# +###################################################################### + + ###################################################################### # # matrix-client-element @@ -4664,6 +4695,8 @@ matrix_synapse_container_additional_networks_auto: | + ([keydb_container_network] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == keydb_identifier else []) + + ([valkey_container_network] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == valkey_identifier else []) + + ([exim_relay_container_network] if (exim_relay_enabled and matrix_synapse_email_enabled and matrix_synapse_email_smtp_host == exim_relay_identifier and matrix_synapse_container_network != exim_relay_container_network) else []) + ([matrix_ma1sd_container_network] if (matrix_ma1sd_enabled and matrix_synapse_account_threepid_delegates_msisdn == matrix_synapse_account_threepid_delegates_msisdn_mas1sd_url and matrix_synapse_container_network != matrix_ma1sd_container_network) else []) @@ -4751,6 +4784,8 @@ matrix_synapse_systemd_required_services_list_auto: | + ([keydb_identifier ~ '.service'] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == keydb_identifier else []) + + ([valkey_identifier ~ '.service'] if matrix_synapse_redis_enabled and matrix_synapse_redis_host == valkey_identifier else []) + + (['matrix-goofys.service'] if matrix_s3_media_store_enabled else []) + (['matrix-authentication-service.service'] if (matrix_authentication_service_enabled and matrix_synapse_experimental_features_msc3861_enabled) else []) @@ -4764,9 +4799,9 @@ matrix_synapse_systemd_wanted_services_list_auto: | }} # Synapse workers (used for parallel load-scaling) need Redis for IPC. -matrix_synapse_redis_enabled: "{{ redis_enabled or keydb_enabled }}" -matrix_synapse_redis_host: "{{ redis_identifier if redis_enabled else (keydb_identifier if keydb_enabled else '') }}" -matrix_synapse_redis_password: "{{ redis_connection_password if redis_enabled else (keydb_connection_password if keydb_enabled else '') }}" +matrix_synapse_redis_enabled: "{{ redis_enabled or keydb_enabled or valkey_enabled }}" +matrix_synapse_redis_host: "{{ valkey_identifier if valkey_enabled else (redis_identifier if redis_enabled else (keydb_identifier if keydb_enabled else '')) }}" +matrix_synapse_redis_password: "{{ valkey_connection_password if valkey_enabled else (redis_connection_password if redis_enabled else (keydb_connection_password if keydb_enabled else '')) }}" matrix_synapse_container_extra_arguments_auto: "{{ matrix_homeserver_container_extra_arguments_auto }}" matrix_synapse_app_service_config_files_auto: "{{ matrix_homeserver_app_service_config_files_auto }}" diff --git a/requirements.yml b/requirements.yml index 2d22bf610..0ed10a4fd 100644 --- a/requirements.yml +++ b/requirements.yml @@ -75,3 +75,6 @@ - src: git+https://github.com/mother-of-all-self-hosting/ansible-role-traefik-certs-dumper.git version: v2.8.3-5 name: traefik_certs_dumper +- src: git+https://github.com/mother-of-all-self-hosting/ansible-role-valkey.git + version: v8.0.1-0 + name: valkey diff --git a/setup.yml b/setup.yml index 51c5589b4..cb5fd7a1a 100644 --- a/setup.yml +++ b/setup.yml @@ -49,6 +49,8 @@ - galaxy/redis - galaxy/keydb + - galaxy/valkey + - role: custom/matrix-authentication-service - custom/matrix-corporal - custom/matrix-appservice-draupnir-for-all