From 8675dedbdb310425d309f6bf1725d7a6c415279e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 19:56:52 +0200 Subject: [PATCH] Add support for automatic (nedb -> Postgres) migration to matrix-appservice-slack --- roles/matrix-base/defaults/main.yml | 1 - .../tasks/migrate_nedb_to_postgres.yml | 11 +++ .../defaults/main.yml | 19 ++-- .../tasks/migrate_nedb_to_postgres.yml | 66 +++++++++++++ .../tasks/setup_install.yml | 37 +++++-- .../tasks/validate_config.yml | 20 ---- .../templates/config.yaml.j2 | 7 +- roles/matrix-postgres/tasks/import_nedb.yml | 98 ------------------- roles/matrix-postgres/tasks/main.yml | 8 -- 9 files changed, 123 insertions(+), 144 deletions(-) create mode 100644 roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml delete mode 100644 roles/matrix-postgres/tasks/import_nedb.yml diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index d0fdcdf86..e0522ba8f 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce run_postgres_import: true run_postgres_upgrade: true run_postgres_import_sqlite_db: true -run_postgres_import_nedb: true run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml index bc6525ec1..3fab195ab 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml @@ -51,3 +51,14 @@ with_items: - rooms.db - users.db + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files." + ] + }} diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index 365493ff2..28646a3c4 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -51,13 +51,18 @@ matrix_appservice_slack_database_password: ~ matrix_appservice_slack_database_hostname: 'matrix-postgres' matrix_appservice_slack_database_port: 5432 matrix_appservice_slack_database_name: matrix_appservice_slack -matrix_appservice_slack_database_file: /data -matrix_appservice_slack_database_connString: >-2 - {%- if matrix_appservice_slack_database_engine == 'postgres' -%} - postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable - {%- elif matrix_appservice_slack_database_engine == 'nedb' -%} - {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }} - {%- endif -%} + +# This is just the Postgres connection string, if Postgres is used. +# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat. +matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable' + +# This is what actually goes into `database.connectionString` for the bridge. +matrix_appservice_slack_database_connectionString: "{{ + { + 'nedb': 'nedb:///data', + 'postgres': matrix_appservice_slack_database_connection_string, + }[matrix_appservice_slack_database_engine] +}}" matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" diff --git a/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml new file mode 100644 index 000000000..fedad9775 --- /dev/null +++ b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml @@ -0,0 +1,66 @@ +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Ensure matrix-appservice-slack is stopped + service: + name: matrix-appservice-slack + state: stopped + +- name: Import appservice-slack NeDB database into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data + --entrypoint=/bin/sh + {{ matrix_appservice_slack_docker_image }} + -c + '/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}' + +- name: Archive NeDB database files + command: + cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup" + with_items: + - teams.db + - room-store.db + - user-store.db + - event-store.db + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files." + ] + }} diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml index 94e0fedf9..29b0f39e9 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml @@ -1,12 +1,5 @@ --- -- name: Ensure Appservice Slack image is pulled - docker_image: - name: "{{ matrix_appservice_slack_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}" - - name: Ensure AppService Slack paths exist file: path: "{{ item }}" @@ -19,6 +12,30 @@ - "{{ matrix_appservice_slack_config_path }}" - "{{ matrix_appservice_slack_data_path }}" +- set_fact: + matrix_appservice_slack_requires_restart: false + +- block: + - name: Check if a nedb database already exists + stat: + path: "{{ matrix_appservice_slack_data_path }}/teams.db" + register: matrix_appservice_slack_nedb_database_path_local_stat_result + + - block: + - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml" + + - set_fact: + matrix_appservice_slack_requires_restart: true + when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_slack_database_engine == 'postgres'" + +- name: Ensure Appservice Slack image is pulled + docker_image: + name: "{{ matrix_appservice_slack_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}" + - name: Ensure Matrix Appservice Slack config installed copy: content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}" @@ -46,3 +63,9 @@ service: daemon_reload: yes when: "matrix_appservice_slack_systemd_service_result.changed" + +- name: Ensure matrix-appservice-slack.service restarted, if necessary + service: + name: "matrix-appservice-slack.service" + state: restarted + when: "matrix_appservice_slack_requires_restart|bool" diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml index 5421b112a..8af10f2f5 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml @@ -10,23 +10,3 @@ - "matrix_appservice_slack_appservice_token" - "matrix_appservice_slack_homeserver_token" - "matrix_appservice_slack_id_token" - -- block: - - name: Check if a neDB database already exists - stat: - path: "{{ matrix_appservice_slack_data_path }}/" - register: matrix_appservice_slack_nedb_stat_result - - - name: Fail if an neDB database already exists when using Postgres - fail: - msg: >-2 - matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/. - It appears that you've been using this bridge with the neDB engine until now. - To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing neDB database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_appservice_slack_nedb_stat_result.stat.exists" - when: "matrix_appservice_slack_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 index 0ae137381..bf8072c15 100644 --- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 @@ -11,9 +11,10 @@ homeserver: {% if matrix_appservice_slack_database_engine == 'nedb' %} dbdir: "/data" -{% endif %} +{% else %} db: - engine: "{{ matrix_appservice_slack_database_engine }}" - connectionString: {{ matrix_appservice_slack_database_connString | to_json }} + engine: {{ matrix_appservice_slack_database_engine|to_json }} + connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }} +{% endif %} matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}" diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml deleted file mode 100644 index 2a3dd5878..000000000 --- a/roles/matrix-postgres/tasks/import_nedb.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- - -# Pre-checks - -- name: Fail if Postgres not enabled - fail: - msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import." - when: "not matrix_postgres_enabled|bool" - -- name: Fail if playbook called incorrectly - fail: - msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars" - when: "nedb_database_path is not defined or nedb_database_path.startswith('<')" - -- name: Check if the provided nedb database file exists - stat: - path: "{{ nedb_database_path }}" - register: nedb_database_path_stat_result - -- name: Fail if provided SQLite database file doesn't exist - fail: - msg: "File cannot be found on the server at {{ nedb_database_path }}" - when: "not nedb_database_path_stat_result.stat.exists" - -# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string, -# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string. - -- block: - - name: Fail if postgres_connection_string_variable_name points to an undefined variable - fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" - when: "postgres_connection_string_variable_name not in vars" - - - name: Get Postgres connection string from variable - set_fact: - postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}" - when: 'postgres_connection_string_variable_name is defined' - -- name: Fail if playbook called incorrectly - fail: - msg: >- - Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`. - Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"` - when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" - - -# Defaults - -- name: Set postgres_start_wait_time, if not provided - set_fact: - postgres_start_wait_time: 15 - when: "postgres_start_wait_time|default('') == ''" - - -# Actual import work - -- name: Ensure matrix-postgres is started - service: - name: matrix-postgres - state: started - daemon_reload: yes - register: matrix_postgres_service_start_result - -- name: Wait a bit, so that Postgres can start - wait_for: - timeout: "{{ postgres_start_wait_time }}" - delegate_to: 127.0.0.1 - become: false - when: "matrix_postgres_service_start_result.changed|bool" - -# No migration.sh available, but found this: -# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts -# Usage should be similar to appservice_irc -- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres - when: database == 'appservice_slack' - command: - cmd: >- - {{ matrix_host_command_docker }} run - --rm - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} - --cap-drop=ALL - --network={{ matrix_docker_network }} - --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro - --entrypoint=/bin/sh - {{ matrix_appservice_slack_docker_image }} - -c - 'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' - - -- name: Inject result - set_fact: - matrix_playbook_runtime_results: | - {{ - matrix_playbook_runtime_results|default([]) - + - [ - "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." - ] - }} diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 86b8f8721..b9c2ae7c9 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -32,14 +32,6 @@ tags: - import-generic-sqlite-db -# Imports slacks neDB to postgres. -- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" - vars: - database: appservice_slack - when: run_postgres_import_nedb|bool - tags: - - import-slack-nedb - - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool tags: