Merge pull request #3441 from Michael-Hollister/michael/mmr-config-updates-7-23-24

Added new fields to MMR config template

docs/configuring-playbook-matrix-media-repo.md

@@ -23,9 +23,11 @@ matrix_media_repo_enabled: true
 # matrix_media_repo_metrics_enabled: true
 ```
 
-The repo is pre-configured for integrating with the Postgres database, NGINX proxy and [Prometheus/Grafana](configuring-playbook-prometheus-grafana.md) (if metrics enabled) from this playbook for all the available homeserver roles. When the media repo is enabled, other media store roles should be disabled (if using Synapse with other media store roles).
+The repo is pre-configured for integrating with the Postgres database, Traefik proxy and [Prometheus/Grafana](configuring-playbook-prometheus-grafana.md) (if metrics enabled) from this playbook for all the available homeserver roles. When the media repo is enabled, other media store roles should be disabled (if using Synapse with other media store roles).
 
-By default, the media-repo will use the local filesystem for data storage. Additional options include `s3` and `IPFS` (experimental). Access token caching is also enabled by default since the logout endpoints are proxied through the media repo.
+By default, the media-repo will use the local filesystem for data storage. You can alternatively use a `s3` cloud backend as well. Access token caching is also enabled by default since the logout endpoints are proxied through the media repo.
+
+**Note:** If you want to use authenticated media endpoints ([MSC3916](https://github.com/matrix-org/matrix-spec-proposals/pull/3916)), you must configure a signing key for your MMR instance to authorize outbound federation requests. See https://docs.t2bot.io/matrix-media-repo/v1.3.5/installation/signing-key/ for more details on how to configure your server with a signing key.
 
 ## Configuring the media-repo
 

group_vars/matrix_servers

@@ -3650,6 +3650,11 @@ matrix_media_repo_homeservers_auto:
     # to "matrix", most functionality requiring the admin API will not work.
     adminApiKind: "{{ 'synapse' if matrix_homeserver_implementation == 'synapse' else ('dendrite' if matrix_homeserver_implementation == 'dendrite' else 'matrix') }}"
 
+    # The signing key to use for authorizing outbound federation requests. If not specified,
+    # requests will not be authorized. See https://docs.t2bot.io/matrix-media-repo/v1.3.5/installation/signing-key/
+    # for details.
+    signingKeyPath: ""
+
 matrix_media_repo_homeserver_federation_enabled: "{{ matrix_homeserver_federation_enabled }}"
 
 ######################################################################

roles/custom/matrix-media-repo/defaults/main.yml

@@ -467,6 +467,11 @@ matrix_media_repo_datastore_s3_opts_bucket_name: "your-media-bucket"
 # before redirection if present).
 matrix_media_repo_datastore_s3_opts_redirect_when_cached: true
 
+# The size of the prefix (path component) to use when storing media in S3. This can
+# help improve download speeds in some S3 providers. Should not be set to higher than
+# 16 to avoid future incompatibilities with MMR. Defaults to zero (no prefix).
+matrix_media_repo_datastore_s3_opts_prefix_length: 0
+
 # Options for controlling archives. Archives are exports of a particular user's content for
 # the purpose of GDPR or moving media to a different server.
 
@@ -742,6 +747,31 @@ matrix_media_repo_rate_limit_requests_per_second: 1
 # The number of requests an IP can send at once before the rate limit is actually considered.
 matrix_media_repo_rate_limit_burst: 10
 
+# The 'leaky bucket' configurations for MMR. Leaky buckets are limited in size and have a slow
+# drain rate, minimizing the ability for a user to consume large amounts of resources.
+#
+# Buckets are checked and applied after the requests per second configuration above. Buckets are
+# disabled when rate limiting is disabled.
+#
+# Note: buckets are *not* shared across processes. If download requests could end up at two different
+# processes, two different buckets may be filled. This behaviour may change in the future.
+
+# The download bucket applies to both download requests and thumbnail requests. Each anonymous
+# user is assigned a single bucket from their IP address. Authenticated requests (when supported)
+# will use the authenticated entity as the subject - either a user or remote server.
+
+# The maximum size of each bucket.
+matrix_media_repo_rate_limit_buckets_download_capacity_bytes: 524288000  # 500mb default
+
+# The number of bytes to "drain" from the bucket every minute.
+matrix_media_repo_rate_limit_buckets_download_drain_bytes_per_minute: 5242880  # 5mb default
+
+# The number of bytes a requester can go over the capacity, once. This is used to give some
+# buffer to allow a single file to be downloaded when the caller is near the limit. This
+# should be set to either your max remote download size or 30% of the capacityBytes, whichever
+# is smaller.
+matrix_media_repo_rate_limit_buckets_download_overflow_limit_bytes: 104857600  # 100mb default (the same as the default remote download maxBytes)
+
 # Identicons are generated avatars for a given username. Some clients use these to give users a
 # default avatar after signing up. Identicons are not part of the official matrix spec, therefore
 # this feature is completely optional.

roles/custom/matrix-media-repo/templates/media-repo/media-repo.yaml.j2

@@ -97,6 +97,11 @@ database:
 #    # admin status. This should be set to one of "synapse", "dendrite", or "matrix". When set
 #    # to "matrix", most functionality requiring the admin API will not work.
 #    adminApiKind: "synapse"
+#
+#    # The signing key to use for authorizing outbound federation requests. If not specified,
+#    # requests will not be authorized. See https://docs.t2bot.io/matrix-media-repo/v1.3.5/installation/signing-key/
+#    # for details.
+#    #signingKeyPath: "/data/example.org.key"
 homeservers:
 {{ matrix_media_repo_homeservers | to_json | from_json
   | to_nice_yaml(indent=2, width=999999, sort_keys=false) | indent(width=2, first=true) }}
@@ -253,6 +258,10 @@ datastores:
       # when `publicBaseUrl` is unset. Defaults to false (cached media will be served by MMR
       # before redirection if present).
       redirectWhenCached: {{ matrix_media_repo_datastore_s3_opts_redirect_when_cached | to_json }}
+      # The size of the prefix (path component) to use when storing media in S3. This can
+      # help improve download speeds in some S3 providers. Should not be set to higher than
+      # 16 to avoid future incompatibilities with MMR. Defaults to zero (no prefix).
+      prefixLength: {{ matrix_media_repo_datastore_s3_opts_prefix_length | to_json }}
 {% endif %}
 
 # Options for controlling archives. Archives are exports of a particular user's content for
@@ -483,6 +492,30 @@ rateLimit:
   # The number of requests an IP can send at once before the rate limit is actually considered.
   burst: {{ matrix_media_repo_rate_limit_burst | to_json }}
 
+  # The 'leaky bucket' configurations for MMR. Leaky buckets are limited in size and have a slow
+  # drain rate, minimizing the ability for a user to consume large amounts of resources.
+  #
+  # Buckets are checked and applied after the requests per second configuration above. Buckets are
+  # disabled when rate limiting is disabled.
+  #
+  # Note: buckets are *not* shared across processes. If download requests could end up at two different
+  # processes, two different buckets may be filled. This behaviour may change in the future.
+  buckets:
+    # The download bucket applies to both download requests and thumbnail requests. Each anonymous
+    # user is assigned a single bucket from their IP address. Authenticated requests (when supported)
+    # will use the authenticated entity as the subject - either a user or remote server.
+    downloads:
+      # The maximum size of each bucket.
+      capacityBytes: {{ matrix_media_repo_rate_limit_buckets_download_capacity_bytes | to_json }} # 500mb default
+      # The number of bytes to "drain" from the bucket every minute.
+      drainBytesPerMinute: {{ matrix_media_repo_rate_limit_buckets_download_drain_bytes_per_minute | to_json }} # 5mb default
+      # The number of bytes a requester can go over the capacity, once. This is used to give some
+      # buffer to allow a single file to be downloaded when the caller is near the limit. This
+      # should be set to either your max remote download size or 30% of the capacityBytes, whichever
+      # is smaller.
+      overflowLimitBytes: {{ matrix_media_repo_rate_limit_buckets_download_overflow_limit_bytes | to_json }} # 100mb default (the same as the default remote download maxBytes)
+
+
 # Identicons are generated avatars for a given username. Some clients use these to give users a
 # default avatar after signing up. Identicons are not part of the official matrix spec, therefore
 # this feature is completely optional.