From 76506f34e0827650afb49233a3605b71a7a1e0a0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Jan 2019 15:37:57 +0200 Subject: [PATCH] Make media-store restore work with server files, not local This is a simplification and a way to make it consistent with how we do Postgres imports (see 6d89319822f4b2ecb), using files coming from the server, not from the local machine. By encouraging people NOT to use local files, we potentially avoid problems such as #34 (Github issue), where people would download `media_store` to their Mac's filesystem and case-sensitivity issues will actually corrupt it. By not encouraging local files usage, it's less likely that people would copy (huge) directories to their local machine like that. --- docs/importing-media-store.md | 22 ++++++++ docs/importing-postgres.md | 2 +- docs/importing-sqlite.md | 2 +- docs/installing.md | 2 +- docs/restoring-media-store.md | 9 ---- .../tasks/import/import_media_store.yml | 51 ++++++++++--------- 6 files changed, 53 insertions(+), 35 deletions(-) create mode 100644 docs/importing-media-store.md delete mode 100644 docs/restoring-media-store.md diff --git a/docs/importing-media-store.md b/docs/importing-media-store.md new file mode 100644 index 000000000..7aa6c5b59 --- /dev/null +++ b/docs/importing-media-store.md @@ -0,0 +1,22 @@ +# Importing `media_store` data files from an existing installation (optional) + +Run this if you'd like to import your `media_store` files from a previous installation of Matrix Synapse. + + +## Prerequisites + +Before doing the actual data restore, **you need to upload your media store directory to the server** (any path is okay). + +If you are [Storing Matrix media files on Amazon S3](configuring-playbook-s3.md) (optional), restoring with this tool is not possible right now. +As an alternative, you can perform a manual restore using the [AWS CLI tool](https://aws.amazon.com/cli/) (e.g. `aws s3 sync /path/to/server/media_store/. s3://name-of-bucket/`) + +**Note for Mac users**: Due to case-sensitivity issues on certain Mac filesystems (HFS or HFS+), filename corruption may occur if you copy a `media_store` directory to your Mac. If you're transferring a `media_store` directory between 2 servers, make sure you do it directly (from server to server with a tool such as [rsync](https://rsync.samba.org/)), and not by downloading the files to your Mac. + + +## Importing + +Run this command (make sure to replace `` with a path on your server): + + ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_media_store=' --tags=import-media-store + +**Note**: `` must be a file path to a `media_store` directory on the server (not on your local machine!). \ No newline at end of file diff --git a/docs/importing-postgres.md b/docs/importing-postgres.md index 9b679bdb3..7fd1ba977 100644 --- a/docs/importing-postgres.md +++ b/docs/importing-postgres.md @@ -1,7 +1,7 @@ # Importing an existing Postgres database from another installation (optional) Run this if you'd like to import your database from a previous installation of Matrix Synapse. -(don't forget to import your `media_store` files as well - see [the restoring media store guide](restoring-media-store.md)). +(don't forget to import your `media_store` files as well - see [the importing-media-store guide](importing-media-store.md)). ## Prerequisites diff --git a/docs/importing-sqlite.md b/docs/importing-sqlite.md index ff60fa324..af705a79f 100644 --- a/docs/importing-sqlite.md +++ b/docs/importing-sqlite.md @@ -1,7 +1,7 @@ # Importing an existing SQLite database from another installation (optional) Run this if you'd like to import your database from a previous default installation of Matrix Synapse. -(don't forget to import your `media_store` files as well - see [the restoring media store guide](restoring-media-store.md)). +(don't forget to import your `media_store` files as well - see [the importing-media-store guide](importing-media-store.md)). While this playbook always sets up PostgreSQL, by default a Matrix Synapse installation would run using an SQLite database. diff --git a/docs/installing.md b/docs/installing.md index 46700ac87..8ca368596 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -21,7 +21,7 @@ After installing, but before starting the services, you may want to do additiona - [Importing an existing Postgres database (from another installation)](importing-postgres.md) (optional) -- [Restoring `media_store` data files from an existing installation](restoring-media-store.md) (optional) +- [Importing `media_store` data files from an existing installation](importing-media-store.md) (optional) ## Starting the services diff --git a/docs/restoring-media-store.md b/docs/restoring-media-store.md deleted file mode 100644 index 62bddcd9a..000000000 --- a/docs/restoring-media-store.md +++ /dev/null @@ -1,9 +0,0 @@ -# Restoring `media_store` data files from an existing installation (optional) - -Run this if you'd like to import your `media_store` files from a previous installation of Matrix Synapse. - -Run this command (make sure to replace `` with a path on your local machine): - - ansible-playbook -i inventory/hosts setup.yml --extra-vars='local_path_media_store=' --tags=import-media-store - -**Note**: `` must be a file path to a `media_store` directory on your local machine (not on the server!). This directory's contents are then copied to the server. \ No newline at end of file diff --git a/roles/matrix-server/tasks/import/import_media_store.yml b/roles/matrix-server/tasks/import/import_media_store.yml index 57dd15cdd..641798352 100644 --- a/roles/matrix-server/tasks/import/import_media_store.yml +++ b/roles/matrix-server/tasks/import/import_media_store.yml @@ -1,49 +1,54 @@ --- +# Pre-checks + - name: Fail if playbook called incorrectly - fail: msg="The `local_path_media_store` variable needs to be provided to this playbook, via --extra-vars" - when: "local_path_media_store is not defined or local_path_media_store.startswith('<')" + fail: msg="The `server_path_media_store` variable needs to be provided to this playbook, via --extra-vars" + when: "server_path_media_store is not defined or server_path_media_store.startswith('<')" + +- name: Fail if media store is on Amazon S3 + fail: msg="Your media store is on Amazon S3. Due to technical limitations, restoring is not supported." + when: "matrix_s3_media_store_enabled" - name: Check if the provided media store directory exists - stat: path="{{ local_path_media_store }}" - delegate_to: 127.0.0.1 - become: false - register: local_path_media_store_stat + stat: path="{{ server_path_media_store }}" + register: server_path_media_store_stat -- name: Fail if provided media store directory doesn't exist on the local machine - fail: msg="{{ local_path_media_store }} cannot be found on the local machine" - when: "not local_path_media_store_stat.stat.exists or not local_path_media_store_stat.stat.isdir" +- name: Fail if provided media store directory doesn't exist on the server + fail: msg="{{ server_path_media_store }} cannot be found on the server" + when: "not server_path_media_store_stat.stat.exists or not server_path_media_store_stat.stat.isdir" - name: Check if media store contains local_content - stat: path="{{ local_path_media_store }}/local_content" - delegate_to: 127.0.0.1 - become: false - register: local_path_media_store_local_content_stat + stat: path="{{ server_path_media_store }}/local_content" + register: server_path_media_store_local_content_stat - name: Check if media store contains remote_content - stat: path="{{ local_path_media_store }}/remote_content" - delegate_to: 127.0.0.1 - become: false - register: local_path_media_store_remote_content_stat + stat: path="{{ server_path_media_store }}/remote_content" + register: server_path_media_store_remote_content_stat - name: Fail if media store directory doesn't look okay (lacking remote and local content) - fail: msg="{{ local_path_media_store }} contains neither local_content nor remote_content. It's most likely a mistake and is not a media store directory." - when: "not local_path_media_store_local_content_stat.stat.exists and not local_path_media_store_remote_content_stat.stat.exists" + fail: msg="{{ server_path_media_store }} contains neither local_content nor remote_content directories. It's most likely a mistake and is not a media store directory." + when: "not server_path_media_store_local_content_stat.stat.exists and not server_path_media_store_remote_content_stat.stat.exists" + + +# Actual import work - name: Ensure matrix-synapse is stopped service: name=matrix-synapse state=stopped daemon_reload=yes register: stopping_result -- name: Ensure provided media store directory is copied to the server +# This can only work with local files, not if the media store is on Amazon S3, +# as it won't be accessible in such a case. +- name: Ensure provided media store directory is synchronized synchronize: - src: "{{ local_path_media_store }}/" + src: "{{ server_path_media_store }}/" dest: "{{ matrix_synapse_media_store_path }}" delete: yes # It's wasteful to preserve owner/group now. We chown below anyway. owner: no group: no - times: "{{ False if matrix_s3_media_store_enabled else True }}" - perms: "{{ False if matrix_s3_media_store_enabled else True }}" + times: yes + delegate_to: "{{ inventory_hostname }}" # This is for the generic case and fails in other cases (remote file systems), # because in such cases the base path (matrix_synapse_media_store_path) is a mount point.