From 2c360a99fe4a66afad6dc03a298bfc80a8769eb3 Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Mon, 22 Jul 2024 17:38:34 -0500 Subject: [PATCH] Added MMR metrics proxying support --- group_vars/matrix_servers | 11 +- .../matrix-media-repo/defaults/main.yml | 17 +++ .../templates/grafana/media-repo.json | 121 +++++++++--------- .../templates/media-repo/labels.j2 | 38 ++++++ 4 files changed, 121 insertions(+), 66 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 1ccaacf1d..5dbe7bc83 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -3611,6 +3611,12 @@ matrix_media_repo_container_labels_traefik_internal_media_entrypoints: "{{ matri matrix_media_repo_container_labels_traefik_internal_matrix_client_media_enabled: "{{ matrix_playbook_internal_matrix_client_api_traefik_entrypoint_enabled }}" matrix_media_repo_container_labels_traefik_internal_matrix_client_media_entrypoints: "{{ matrix_playbook_internal_matrix_client_api_traefik_entrypoint_name }}" +matrix_media_repo_metrics_proxying_enabled: "{{ matrix_media_repo_metrics_enabled and matrix_metrics_exposure_enabled }}" +matrix_media_repo_metrics_proxying_hostname: "{{ matrix_metrics_exposure_hostname }}" +matrix_media_repo_metrics_proxying_path: "{{ matrix_metrics_exposure_path_prefix }}/matrix-media-repo" +matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_enabled: "{{ matrix_metrics_exposure_http_basic_auth_enabled }}" +matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_users: "{{ matrix_metrics_exposure_http_basic_auth_users }}" + matrix_media_repo_database_hostname: "{{ devture_postgres_connection_hostname if devture_postgres_enabled else '' }}" matrix_media_repo_database_username: matrix_media_repo matrix_media_repo_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'mediarepo.db', rounds=655555) | to_uuid }}" @@ -4880,11 +4886,6 @@ grafana_provisioning_dashboard_template_files: | 'path': 'roles/custom/matrix-prometheus-nginxlog-exporter/templates/grafana/nginx-proxy.json', 'name': 'nginx-proxy.json', }] if matrix_prometheus_nginxlog_exporter_enabled else []) - + - ([{ - 'path': 'roles/custom/matrix-media-repo/templates/grafana/media-repo.json', - 'name': 'media-repo.json', - }] if matrix_media_repo_metrics_enabled else []) }} grafana_default_home_dashboard_path: |- diff --git a/roles/custom/matrix-media-repo/defaults/main.yml b/roles/custom/matrix-media-repo/defaults/main.yml index 1ac4d6b56..1c5bfd665 100755 --- a/roles/custom/matrix-media-repo/defaults/main.yml +++ b/roles/custom/matrix-media-repo/defaults/main.yml @@ -44,6 +44,11 @@ matrix_media_repo_container_network: "{{ matrix_media_repo_identifier }}" # Use this to expose this container to another reverse proxy, which runs in a different container network. matrix_media_repo_container_additional_networks: [] +# Controls whether media repo metrics should be proxied (exposed) on `matrix.DOMAIN/metrics/matrix-media-repo` +matrix_media_repo_metrics_proxying_enabled: false +matrix_media_repo_metrics_proxying_hostname: "" +matrix_media_repo_metrics_proxying_path: "/metrics/matrix-media-repo" + # Controls whether the matrix-media-repo container exposes its HTTP port (tcp/8000 in the container). # # Takes an ":" or "" value (e.g. "127.0.0.1:8000"), or empty string to not expose. @@ -134,6 +139,18 @@ matrix_media_repo_container_labels_traefik_t2bot_entrypoints: "{{ matrix_media_r matrix_media_repo_container_labels_traefik_t2bot_tls: "{{ matrix_media_repo_container_labels_traefik_t2bot_entrypoints != 'web' }}" matrix_media_repo_container_labels_traefik_t2bot_tls_certResolver: default # noqa var-naming +# Controls whether labels will be added that expose the media repo metrics endpoint +matrix_media_repo_container_labels_traefik_metrics_enabled: "{{ matrix_media_repo_metrics_enabled and matrix_media_repo_metrics_proxying_enabled }}" +matrix_media_repo_container_labels_traefik_metrics_rule: "Host(`{{ matrix_media_repo_metrics_proxying_hostname }}`) && PathPrefix(`{{ matrix_media_repo_metrics_proxying_path }}`)" +matrix_media_repo_container_labels_traefik_metrics_priority: 0 +matrix_media_repo_container_labels_traefik_metrics_entrypoints: "{{ matrix_media_repo_container_labels_traefik_entrypoints }}" +matrix_media_repo_container_labels_traefik_metrics_tls: "{{ matrix_media_repo_container_labels_traefik_t2bot_entrypoints != 'web' }}" +matrix_media_repo_container_labels_traefik_metrics_tls_certResolver: default # noqa var-naming + +matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_enabled: false +# See: https://doc.traefik.io/traefik/middlewares/http/basicauth/#users +matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_users: '' + # Traefik labels handling the old `/_matrix/media` endpoints on the federation entrypint. # These are being superseded by `/_matrix/federation/VERSION/media` endpoints - see `matrix_media_repo_container_labels_traefik_federation_matrix_federation_media_*`. matrix_media_repo_container_labels_traefik_media_federation_enabled: true diff --git a/roles/custom/matrix-media-repo/templates/grafana/media-repo.json b/roles/custom/matrix-media-repo/templates/grafana/media-repo.json index feee5bace..b2c2eadfa 100644 --- a/roles/custom/matrix-media-repo/templates/grafana/media-repo.json +++ b/roles/custom/matrix-media-repo/templates/grafana/media-repo.json @@ -1,59 +1,16 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.1.0" - }, - { - "type": "panel", - "id": "heatmap", - "name": "Heatmap", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], "annotations": { "list": [ { "builtIn": 1, "datasource": { - "type": "datasource", - "uid": "grafana" + "type": "grafana", + "uid": "${DS_PROMETHEUS}" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, "type": "dashboard" } ] @@ -153,7 +110,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(media_http_requests_total[2m])", + "expr": "rate(media_http_requests_total{host=\"$host\"}[2m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ '{{host}}: {{method}} {{action}}' }}", @@ -166,7 +123,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(media_invalid_http_requests_total[2m])", + "expr": "rate(media_invalid_http_requests_total{host=\"$host\"}[2m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ 'Invalid Host: {{method}} {{action}}' }}", @@ -265,7 +222,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(media_http_responses_total[2m])", + "expr": "rate(media_http_responses_total{host=\"$host\"}[2m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ '{{host}}: {{method}} {{action}} {{statusCode}}' }}", @@ -278,7 +235,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(media_invalid_http_requests_total[2m])", + "expr": "rate(media_invalid_http_requests_total{host=\"$host\"}[2m])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -369,7 +326,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(media_http_response_time_seconds_bucket{action=~\"download|thumbnail\"}[2m])) by (le)", + "expr": "sum(rate(media_http_response_time_seconds_bucket{action=~\"download|thumbnail\", host=\"$host\"}[2m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -460,7 +417,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(media_http_response_time_seconds_bucket{action=\"upload\"}[2m])) by (le)", + "expr": "sum(rate(media_http_response_time_seconds_bucket{action=\"upload\", host=\"$host\"}[2m])) by (le)", "format": "heatmap", "instant": false, "legendFormat": "{{ '{{method}} {{action}} - {{le}}' }}", @@ -560,7 +517,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "go_memstats_alloc_bytes{job=\"media_repo\"}", + "expr": "go_memstats_alloc_bytes", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -574,7 +531,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "go_memstats_sys_bytes{job=\"media_repo\"}", + "expr": "go_memstats_sys_bytes", "interval": "", "legendFormat": "memory usage (sys)", "refId": "C" @@ -585,7 +542,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "go_memstats_heap_alloc_bytes{job=\"media_repo\"}", + "expr": "go_memstats_heap_alloc_bytes", "interval": "", "legendFormat": "heap usage (alloc)", "refId": "A" @@ -596,7 +553,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "go_memstats_heap_idle_bytes{job=\"media_repo\"}", + "expr": "go_memstats_heap_idle_bytes", "interval": "", "legendFormat": "heap usage (idle)", "refId": "D" @@ -607,7 +564,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "go_memstats_heap_inuse_bytes{job=\"media_repo\"}", + "expr": "go_memstats_heap_inuse_bytes", "interval": "", "legendFormat": "heap usage (used)", "refId": "E" @@ -619,7 +576,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "go_memstats_heap_released_bytes{job=\"media_repo\"}", + "expr": "go_memstats_heap_released_bytes", "hide": false, "legendFormat": "heap usage (released)", "range": true, @@ -718,7 +675,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "go_goroutines{job=\"media_repo\"}", + "expr": "go_goroutines", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -733,7 +690,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "go_threads{job=\"media_repo\"}", + "expr": "go_threads", "format": "time_series", "hide": false, "interval": "", @@ -1263,9 +1220,51 @@ "refresh": "1m", "schemaVersion": 38, "style": "dark", - "tags": [], + "tags": [ + "matrix" + ], "templating": { - "list": [] + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(media_http_response_time_seconds_bucket,host)", + "hide": 0, + "includeAll": false, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(media_http_response_time_seconds_bucket,host)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] }, "time": { "from": "now-1h", diff --git a/roles/custom/matrix-media-repo/templates/media-repo/labels.j2 b/roles/custom/matrix-media-repo/templates/media-repo/labels.j2 index e6b675880..57b21d897 100755 --- a/roles/custom/matrix-media-repo/templates/media-repo/labels.j2 +++ b/roles/custom/matrix-media-repo/templates/media-repo/labels.j2 @@ -6,6 +6,7 @@ traefik.docker.network={{ matrix_media_repo_container_labels_traefik_docker_netw {% endif %} traefik.http.services.matrix-media-repo.loadbalancer.server.port={{ matrix_media_repo_port }} +traefik.http.services.matrix-media-repo-metrics.loadbalancer.server.port={{ matrix_media_repo_metrics_port }} {% set middlewares = [] %} @@ -243,6 +244,43 @@ traefik.http.routers.matrix-media-repo-public-t2bot.tls.certResolver={{ matrix_m {% endif %} +{% if matrix_media_repo_container_labels_traefik_metrics_enabled %} +############################################################ +# # +# Metrics # +# # +############################################################ + +{% set metricsMiddlewares = ['matrix-media-repo-metrics-replace-path'] %} +traefik.http.middlewares.matrix-media-repo-metrics-replace-path.replacepath.path=/metrics + +{% if matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_enabled %} +{% set metricsMiddlewares = metricsMiddlewares + ['matrix-media-repo-metrics-basic-auth'] %} +traefik.http.middlewares.matrix-media-repo-metrics-basic-auth.basicauth.users={{ matrix_media_repo_container_labels_traefik_metrics_middleware_basic_auth_users }} +{% endif %} + +traefik.http.routers.matrix-media-repo-metrics.rule={{ matrix_media_repo_container_labels_traefik_metrics_rule }} +traefik.http.routers.matrix-media-repo-metrics.middlewares={{ metricsMiddlewares | join(',') }} + +{% if matrix_media_repo_container_labels_traefik_metrics_priority | int > 0 %} +traefik.http.routers.matrix-media-repo-metrics.priority={{ matrix_media_repo_container_labels_traefik_metrics_priority }} +{% endif %} + +traefik.http.routers.matrix-media-repo-metrics.service=matrix-media-repo-metrics +traefik.http.routers.matrix-media-repo-metrics.entrypoints={{ matrix_media_repo_container_labels_traefik_metrics_entrypoints }} + +traefik.http.routers.matrix-media-repo-metrics.tls={{ matrix_media_repo_container_labels_traefik_metrics_tls | to_json }} +{% if matrix_media_repo_container_labels_traefik_metrics_tls %} +traefik.http.routers.matrix-media-repo-metrics.tls.certResolver={{ matrix_media_repo_container_labels_traefik_metrics_tls_certResolver }} +{% endif %} + +############################################################ +# # +# /Metrics # +# # +############################################################ +{% endif %} + {% if matrix_media_repo_homeserver_federation_enabled %} # Matrix Federation