From d17fac035d7d76c13b88c57a502476ced754298f Mon Sep 17 00:00:00 2001 From: Astra Logical Date: Sun, 2 Aug 2026 17:56:01 -0400 Subject: [PATCH] Add matrix alertmanager receiver with matrix-alertmanager bot. Fix corrupted jellyfin cache pvc --- ...-services.yaml => monitoring-ingress.yaml} | 0 .../monitoring-matrix-alertmanager.yaml | 98 +++++++++++++++++++ monitoring/monitoring-stack.yaml | 32 ++++-- services/jellyfin/jellyfin-deployment.yaml | 2 +- services/jellyfin/jellyfin-pvcs.yaml | 2 +- 5 files changed, 123 insertions(+), 11 deletions(-) rename monitoring/{monitoring-services.yaml => monitoring-ingress.yaml} (100%) create mode 100644 monitoring/monitoring-matrix-alertmanager.yaml diff --git a/monitoring/monitoring-services.yaml b/monitoring/monitoring-ingress.yaml similarity index 100% rename from monitoring/monitoring-services.yaml rename to monitoring/monitoring-ingress.yaml diff --git a/monitoring/monitoring-matrix-alertmanager.yaml b/monitoring/monitoring-matrix-alertmanager.yaml new file mode 100644 index 0000000..bcb940a --- /dev/null +++ b/monitoring/monitoring-matrix-alertmanager.yaml @@ -0,0 +1,98 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: matrix-alertmanager-secret + namespace: monitoring +spec: + refreshInterval: "1h" + secretStoreRef: + name: vault-external-secrets-store + kind: ClusterSecretStore + target: + name: matrix-alertmanager-secret + creationPolicy: Owner + template: + data: + MATRIX_USER: "{{ .MATRIX_USER }}" + MATRIX_TOKEN: "{{ .MATRIX_TOKEN }}" + APP_ALERTMANAGER_SECRET: "{{ .APP_ALERTMANAGER_SECRET }}" + WEBHOOK_URL: "http://matrix-alertmanager.monitoring.svc.cluster.local:3000/alerts?secret={{ .APP_ALERTMANAGER_SECRET }}" + data: + - secretKey: MATRIX_USER + remoteRef: + key: matrix-alertmanager + property: matrix-user + - secretKey: MATRIX_TOKEN + remoteRef: + key: matrix-alertmanager + property: matrix-token + - secretKey: APP_ALERTMANAGER_SECRET + remoteRef: + key: matrix-alertmanager + property: secret-key +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: matrix-alertmanager-config + namespace: monitoring +data: + MATRIX_HOMESERVER_URL: "https://starling.mynest.love" + APP_PORT: "3000" + MATRIX_ROOMS: "matrix-notifications/!mlnAreDcLexiphBSel:starling.mynest.love" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matrix-alertmanager + namespace: monitoring + labels: + app.kubernetes.io/name: matrix-alertmanager + annotations: + keel.sh/policy: "all" + keel.sh/match-tag: "true" + keel.sh/schedule: "0 3 * * *" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: matrix-alertmanager + template: + metadata: + labels: + app.kubernetes.io/name: matrix-alertmanager + spec: + nodeSelector: + kubernetes.io/arch: amd64 # No ARM image + containers: + - name: matrix-alertmanager + image: jaywink/matrix-alertmanager:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http + envFrom: + - secretRef: + name: matrix-alertmanager-secret + - configMapRef: + name: matrix-alertmanager-config + env: + - name: MENTION_ROOM + value: "1" +--- +apiVersion: v1 +kind: Service +metadata: + name: matrix-alertmanager + namespace: monitoring + labels: + app.kubernetes.io/name: matrix-alertmanager +spec: + type: ClusterIP + ports: + - port: 3000 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: matrix-alertmanager diff --git a/monitoring/monitoring-stack.yaml b/monitoring/monitoring-stack.yaml index 2847911..592de98 100644 --- a/monitoring/monitoring-stack.yaml +++ b/monitoring/monitoring-stack.yaml @@ -16,6 +16,23 @@ spec: kubeProxy: enabled: false + # Filter out IPV6 monitors + kubelet: + enabled: true + serviceMonitor: + cAdvisorRelabelings: + - sourceLabels: [__address__] + regex: '^\[.*' + action: drop + probesRelabelings: + - sourceLabels: [__address__] + regex: '^\[.*' + action: drop + relabelings: + - sourceLabels: [__address__] + regex: '^\[.*' + action: drop + prometheus: prometheusSpec: serviceMonitorSelectorNilUsesHelmValues: false @@ -44,7 +61,7 @@ spec: group_wait: 30s group_interval: 5m repeat_interval: 12h - receiver: 'discord-notifications' + receiver: 'matrix-notifications' routes: - matchers: - alertname = Watchdog @@ -55,16 +72,13 @@ spec: equal: ['alertname', 'dev', 'instance'] receivers: - name: 'null' - - name: 'discord-notifications' - discord_configs: - - webhook_url: 'https://discord.com/api/webhooks/1483240068197126300/zI_k1nEc8U8Fd0hiJc7gBG0TjklblmYrSDvpbTMiposGc32_0UREpLjch1FggFvluV3D' + - name: 'matrix-notifications' + webhook_configs: + - url_file: '/etc/alertmanager/secrets/matrix-alertmanager-secret/WEBHOOK_URL' send_resolved: true - title: '{{ .Status | toUpper }}: {{ .GroupLabels.alertname }}' - message: |- - **Summary:** {{ range .Alerts }}{{ .Annotations.summary }}{{ end }} - **Description:** {{ range .Alerts }}{{ .Annotations.description }}{{ end }} - **Severity:** {{ range .Alerts }}{{ .Labels.severity }}{{ end }} alertmanagerSpec: + secrets: + - matrix-alertmanager-secret externalUrl: http://alertmanager.k3s.home.nest storage: volumeClaimTemplate: diff --git a/services/jellyfin/jellyfin-deployment.yaml b/services/jellyfin/jellyfin-deployment.yaml index b0f519a..d35cb02 100644 --- a/services/jellyfin/jellyfin-deployment.yaml +++ b/services/jellyfin/jellyfin-deployment.yaml @@ -44,7 +44,7 @@ spec: claimName: jellyfin-config-pvc - name: jellyfin-cache persistentVolumeClaim: - claimName: jellyfin-cache-pvc + claimName: jellyfin-cache-backup - name: yarr nfs: server: robin.home.nest diff --git a/services/jellyfin/jellyfin-pvcs.yaml b/services/jellyfin/jellyfin-pvcs.yaml index 87611c2..e217bea 100644 --- a/services/jellyfin/jellyfin-pvcs.yaml +++ b/services/jellyfin/jellyfin-pvcs.yaml @@ -16,7 +16,7 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: jellyfin-cache-pvc + name: jellyfin-cache-backup labels: recurring-job.longhorn.io/source: "enabled" recurring-job-group.longhorn.io/app-config-group: "enabled"