add synapse_auto_compressor

This commit is contained in:
Aine 2023-03-10 23:48:43 +02:00
parent 1bdd3e2503
commit bf3677a9b5
No known key found for this signature in database
GPG Key ID: 34969C908CCA2804
9 changed files with 232 additions and 0 deletions

View File

@ -163,6 +163,7 @@ Various services that don't fit any other category.
| Name | Default? | Description | Documentation | | Name | Default? | Description | Documentation |
| ---- | -------- | ----------- | ------------- | | ---- | -------- | ----------- | ------------- |
| [sliding-sync](https://github.com/matrix-org/sliding-sync)| x | Sliding Sync support for clients which require it (e.g. Element X) | [Link](docs/configuring-playbook-sliding-sync-proxy.md) | | [sliding-sync](https://github.com/matrix-org/sliding-sync)| x | Sliding Sync support for clients which require it (e.g. Element X) | [Link](docs/configuring-playbook-sliding-sync-proxy.md) |
| [synapse_auto_compressor](https://github.com/matrix-org/rust-synapse-compress-state/#automated-tool-synapse_auto_compressor) | x | A cli tool that automatically compresses `state_groups` database table in background. | [Link](docs/configuring-playbook-synapse-auto-compressor.md) |
| [synapse-simple-antispam](https://github.com/t2bot/synapse-simple-antispam) (advanced) | x | A spam checker module | [Link](docs/configuring-playbook-synapse-simple-antispam.md) | | [synapse-simple-antispam](https://github.com/t2bot/synapse-simple-antispam) (advanced) | x | A spam checker module | [Link](docs/configuring-playbook-synapse-simple-antispam.md) |
| [Matrix Corporal](https://github.com/devture/matrix-corporal) (advanced) | x | Reconciliator and gateway for a managed Matrix server | [Link](docs/configuring-playbook-matrix-corporal.md) | | [Matrix Corporal](https://github.com/devture/matrix-corporal) (advanced) | x | Reconciliator and gateway for a managed Matrix server | [Link](docs/configuring-playbook-matrix-corporal.md) |
| [Etherpad](https://etherpad.org) | x | An open source collaborative text editor | [Link](docs/configuring-playbook-etherpad.md) | | [Etherpad](https://etherpad.org) | x | An open source collaborative text editor | [Link](docs/configuring-playbook-etherpad.md) |

View File

@ -0,0 +1,36 @@
# Setting up synapse_auto_compressor
The playbook can install and configure [synapse_auto_compressor](https://github.com/matrix-org/rust-synapse-compress-state/#automated-tool-synapse_auto_compressor) for you.
It's a cli tool that automatically compresses `state_groups` database table in background.
See the project's [documentation](https://github.com/matrix-org/rust-synapse-compress-state/#automated-tool-synapse_auto_compressor) to learn what it does and why it might be useful to you.
## Adjusting the playbook configuration
Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file:
```yaml
matrix_synapse_auto_compressor_enabled: true
```
## Installing
After configuring the playbook, run the [installation](installing.md) command again:
```
ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start
```
## Usage
After installation, synapse_auto_compressor will run automatically every day by the defined schedule
## Manually start the tool
For testing your setup it can be helpful to not wait until 00:00. If you want to run the tool immediately, log onto the server
and run `systemctl start matrix-synapse-auto-compressor`. This will not return until the run is done, so possibly a long time.
Consider using [tmux](https://en.wikipedia.org/wiki/Tmux) if your SSH connection is unstable.

View File

@ -84,6 +84,7 @@
- custom/matrix-cactus-comments - custom/matrix-cactus-comments
- custom/matrix-rageshake - custom/matrix-rageshake
- custom/matrix-synapse - custom/matrix-synapse
- custom/matrix-synapse-auto-compressor
- custom/matrix-synapse-reverse-proxy-companion - custom/matrix-synapse-reverse-proxy-companion
- custom/matrix-dendrite - custom/matrix-dendrite
- custom/matrix-conduit - custom/matrix-conduit

View File

@ -0,0 +1,51 @@
---
# synapse_auto_compressor tool
# Project source code URL: https://github.com/matrix-org/rust-synapse-compress-state
matrix_synapse_auto_compressor_enabled: false
matrix_synapse_auto_compressor_container_image_self_build: false
matrix_synapse_auto_compressor_container_repo: "https://gitlab.com/etke.cc/rust-synapse-compress-state.git"
matrix_synapse_auto_compressor_container_repo_version: "{{ 'main' if matrix_synapse_auto_compressor_version == 'latest' else matrix_synapse_auto_compressor_version }}"
matrix_synapse_auto_compressor_container_src_files_path: "{{ matrix_synapse_auto_compressor_base_path }}"
matrix_synapse_auto_compressor_version: v0.1.3
matrix_synapse_auto_compressor_container_image: "{{ matrix_synapse_auto_compressor_container_image_name_prefix }}etke.cc/rust-synapse-compress-state:{{ matrix_synapse_auto_compressor_version }}"
matrix_synapse_auto_compressor_container_image_name_prefix: "{{ 'localhost/' if matrix_synapse_auto_compressor_container_image_self_build else 'registry.gitlab.com/' }}"
matrix_synapse_auto_compressor_container_image_force_pull: "{{ matrix_synapse_auto_compressor_container_image.endswith(':latest') }}"
matrix_synapse_auto_compressor_base_path: "{{ matrix_base_data_path }}/synapse-auto-compressor"
# A list of extra arguments to pass to the container
matrix_synapse_auto_compressor_container_extra_arguments: []
# List of systemd services that matrix-synapse-auto-compressor.service depends on
matrix_synapse_auto_compressor_systemd_required_services_list: ['docker.service']
# List of systemd services that matrix-synapse-auto-compressor.service wants
matrix_synapse_auto_compressor_systemd_wanted_services_list: []
matrix_synapse_auto_compressor_database_username: 'synapse'
matrix_synapse_auto_compressor_database_password: 'some-password'
matrix_synapse_auto_compressor_database_hostname: ''
matrix_synapse_auto_compressor_database_port: 5432
matrix_synapse_auto_compressor_database_name: 'synapse'
# connection string to synapse database (postgres only)
matrix_synapse_auto_compressor_synapse_database: 'postgres://{{ matrix_synapse_auto_compressor_database_username }}:{{ matrix_synapse_auto_compressor_database_password }}@{{ matrix_synapse_auto_compressor_database_hostname }}:{{ matrix_synapse_auto_compressor_database_port }}/{{ matrix_synapse_auto_compressor_database_name }}'
# systemd calendar configuration for the compressor job
matrix_synapse_auto_compressor_calendar: "*-*-* 00:00:00"
# The number of state groups to work on at once.
# All of the entries from state_groups_state are requested from the database for state groups that are worked on.
# Therefore small chunk sizes may be needed on machines with low memory.
# Note: if the compressor fails to find space savings on the chunk as a whole
# (which may well happen in rooms with lots of backfill in) then the entire chunk is skipped.
matrix_synapse_auto_compressor_chunk_size: 500
# CHUNKS_TO_COMPRESS chunks of size CHUNK_SIZE will be compressed.
# The higher this number is set to, the longer the compressor will run for.
matrix_synapse_auto_compressor_chunks_to_compress: 100
matrix_synapse_auto_compressor_command: "synapse_auto_compressor -p {{ matrix_synapse_auto_compressor_synapse_database }} -c {{ matrix_synapse_auto_compressor_chunk_size }} -n {{ matrix_synapse_auto_compressor_chunks_to_compress }}"

View File

@ -0,0 +1,53 @@
---
- name: Ensure synapse-auto-compressor paths exist
ansible.builtin.file:
path: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
state: directory
mode: 0750
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
when: matrix_synapse_auto_compressor_container_image_self_build | bool
- name: Ensure synapse-auto-compressor image is pulled
community.docker.docker_image:
name: "{{ matrix_synapse_auto_compressor_container_image }}"
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
force_source: "{{ matrix_synapse_auto_compressor_container_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_synapse_auto_compressor_container_image_force_pull }}"
when: "not matrix_synapse_auto_compressor_container_image_self_build | bool"
register: result
retries: "{{ devture_playbook_help_container_retries_count }}"
delay: "{{ devture_playbook_help_container_retries_delay }}"
until: result is not failed
- name: Ensure synapse-auto-compressor repository is present on self-build
ansible.builtin.git:
repo: "{{ matrix_synapse_auto_compressor_container_repo }}"
version: "{{ matrix_synapse_auto_compressor_container_repo_version }}"
dest: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
force: "yes"
become: true
become_user: "{{ matrix_user_username }}"
register: matrix_synapse_auto_compressor_git_pull_results
when: "matrix_synapse_auto_compressor_container_image_self_build | bool"
- name: Ensure synapse-auto-compressor image is built
community.docker.docker_image:
name: "{{ matrix_synapse_auto_compressor_container_image }}"
source: build
force_source: "{{ matrix_synapse_auto_compressor_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mailer_git_pull_results.changed }}"
build:
dockerfile: Dockerfile
path: "{{ matrix_synapse_auto_compressor_container_src_files_path }}"
pull: true
when: "matrix_synapse_auto_compressor_container_image_self_build | bool"
- name: Ensure matrix-synapse-auto-compressor systemd service and timer are installed
ansible.builtin.template:
src: "{{ role_path }}/templates/matrix-synapse-auto-compressor.{{ item }}.j2"
dest: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.{{ item }}"
mode: 0644
with_items:
- service
- timer

View File

@ -0,0 +1,17 @@
---
- block:
- when: matrix_synapse_auto_compressor_enabled | bool
ansible.builtin.include_tasks: "{{ role_path }}/tasks/install.yml"
tags:
- setup-all
- setup-synapse-auto-compressor
- install-all
- install-synapse-auto-compressor
- block:
- when: not matrix_synapse_auto_compressor_enabled | bool
ansible.builtin.include_tasks: "{{ role_path }}/tasks/uninstall.yml"
tags:
- setup-all
- setup-synapse-auto-compressor

View File

@ -0,0 +1,29 @@
---
- name: Check existence of matrix-synapse-auto-compressor service
ansible.builtin.stat:
path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.service"
register: matrix_synapse_auto_compressor_service_stat
- when: matrix_synapse_auto_compressor_service_stat.stat.exists | bool
block:
- name: Ensure matrix-synapse-auto-compressor is stopped
ansible.builtin.service:
name: matrix-synapse-auto-compressor
state: stopped
enabled: false
daemon_reload: true
- name: Ensure matrix-synapse-auto-compressor.service doesn't exist
ansible.builtin.file:
path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.service"
state: absent
- name: Ensure matrix-synapse-auto-compressor.timer doesn't exist
ansible.builtin.file:
path: "{{ devture_systemd_docker_base_systemd_path }}/matrix-synapse-auto-compressor.timer"
state: absent
- name: Ensure Matrix synapse-auto-compressor paths don't exist
ansible.builtin.file:
path: "{{ matrix_synapse_auto_compressor_base_path }}"
state: absent

View File

@ -0,0 +1,35 @@
#jinja2: lstrip_blocks: "True"
[Unit]
Description=Synapse State Auto Compressor
{% for service in matrix_synapse_auto_compressor_systemd_required_services_list %}
Requires={{ service }}
After={{ service }}
{% endfor %}
{% for service in matrix_synapse_auto_compressor_systemd_wanted_services_list %}
Wants={{ service }}
{% endfor %}
DefaultDependencies=no
[Service]
Type=oneshot
Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-synapse-auto-compressor 2>/dev/null || true'
ExecStartPre=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-synapse-auto-compressor 2>/dev/null || true'
ExecStart={{ devture_systemd_docker_base_host_command_docker }} run --rm --name matrix-synapse-auto-compressor \
--log-driver=none \
--cap-drop=ALL \
--read-only \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
--network={{ matrix_docker_network }} \
{% for arg in matrix_synapse_auto_compressor_container_extra_arguments %}
{{ arg }} \
{% endfor %}
{{ matrix_synapse_auto_compressor_container_image }} \
{{ matrix_synapse_auto_compressor_command }}
ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} kill matrix-synapse-auto-compressor 2>/dev/null || true'
ExecStop=-{{ devture_systemd_docker_base_host_command_sh }} -c '{{ devture_systemd_docker_base_host_command_docker }} rm matrix-synapse-auto-compressor 2>/dev/null || true'
SyslogIdentifier=matrix-synapse-auto-compressor
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,9 @@
[Unit]
Description=Synapse State Auto Compressor Timer
[Timer]
Unit=matrix-synapse-auto-compressor.service
OnCalendar={{ matrix_synapse_auto_compressor_calendar }}
[Install]
WantedBy=timers.target