Add matrix alertmanager receiver with matrix-alertmanager bot. Fix corrupted jellyfin cache pvc

monitoring/monitoring-matrix-alertmanager.yaml

@@ -0,0 +1,98 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: matrix-alertmanager-secret
+  namespace: monitoring
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    name: vault-external-secrets-store
+    kind: ClusterSecretStore
+  target:
+    name: matrix-alertmanager-secret
+    creationPolicy: Owner
+    template:
+      data:
+        MATRIX_USER: "{{ .MATRIX_USER }}"
+        MATRIX_TOKEN: "{{ .MATRIX_TOKEN }}"
+        APP_ALERTMANAGER_SECRET: "{{ .APP_ALERTMANAGER_SECRET }}"
+        WEBHOOK_URL: "http://matrix-alertmanager.monitoring.svc.cluster.local:3000/alerts?secret={{ .APP_ALERTMANAGER_SECRET }}"
+  data:
+  - secretKey: MATRIX_USER
+    remoteRef:
+      key: matrix-alertmanager
+      property: matrix-user
+  - secretKey: MATRIX_TOKEN
+    remoteRef:
+      key: matrix-alertmanager
+      property: matrix-token
+  - secretKey: APP_ALERTMANAGER_SECRET
+    remoteRef:
+      key: matrix-alertmanager
+      property: secret-key
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: matrix-alertmanager-config
+  namespace: monitoring
+data:
+  MATRIX_HOMESERVER_URL: "https://starling.mynest.love"
+  APP_PORT: "3000"
+  MATRIX_ROOMS: "matrix-notifications/!mlnAreDcLexiphBSel:starling.mynest.love"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: matrix-alertmanager
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: matrix-alertmanager
+  annotations:
+    keel.sh/policy: "all"
+    keel.sh/match-tag: "true"
+    keel.sh/schedule: "0 3 * * *"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: matrix-alertmanager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: matrix-alertmanager
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: amd64 # No ARM image
+      containers:
+        - name: matrix-alertmanager
+          image: jaywink/matrix-alertmanager:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 3000
+              name: http
+          envFrom:
+            - secretRef:
+                name: matrix-alertmanager-secret
+            - configMapRef:
+                name: matrix-alertmanager-config
+          env:
+            - name: MENTION_ROOM
+              value: "1"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: matrix-alertmanager
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: matrix-alertmanager
+spec:
+  type: ClusterIP
+  ports:
+    - port: 3000
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app.kubernetes.io/name: matrix-alertmanager

monitoring/monitoring-stack.yaml

@@ -16,6 +16,23 @@ spec:
     kubeProxy:
       enabled: false
 
+    # Filter out IPV6 monitors
+    kubelet:
+      enabled: true
+      serviceMonitor:
+        cAdvisorRelabelings:
+          - sourceLabels: [__address__]
+            regex: '^\[.*'
+            action: drop
+        probesRelabelings:
+          - sourceLabels: [__address__]
+            regex: '^\[.*'
+            action: drop
+        relabelings:
+          - sourceLabels: [__address__]
+            regex: '^\[.*'
+            action: drop
+
     prometheus:
       prometheusSpec:
         serviceMonitorSelectorNilUsesHelmValues: false
@@ -44,7 +61,7 @@ spec:
           group_wait: 30s
           group_interval: 5m
           repeat_interval: 12h 
-          receiver: 'discord-notifications'
+          receiver: 'matrix-notifications'
           routes:
             - matchers:
                 - alertname = Watchdog
@@ -55,16 +72,13 @@ spec:
             equal: ['alertname', 'dev', 'instance']
         receivers:
           - name: 'null'
-          - name: 'discord-notifications'
-            discord_configs:
-              - webhook_url: 'https://discord.com/api/webhooks/1483240068197126300/zI_k1nEc8U8Fd0hiJc7gBG0TjklblmYrSDvpbTMiposGc32_0UREpLjch1FggFvluV3D'
+          - name: 'matrix-notifications'
+            webhook_configs:
+              - url_file: '/etc/alertmanager/secrets/matrix-alertmanager-secret/WEBHOOK_URL'
                 send_resolved: true
-                title: '{{ .Status | toUpper }}: {{ .GroupLabels.alertname }}'
-                message: |-
-                  **Summary:** {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
-                  **Description:** {{ range .Alerts }}{{ .Annotations.description }}{{ end }}
-                  **Severity:** {{ range .Alerts }}{{ .Labels.severity }}{{ end }}
       alertmanagerSpec:
+        secrets:
+          - matrix-alertmanager-secret
         externalUrl: http://alertmanager.k3s.home.nest
         storage:
           volumeClaimTemplate:

services/jellyfin/jellyfin-deployment.yaml

@@ -44,7 +44,7 @@ spec:
             claimName: jellyfin-config-pvc
         - name: jellyfin-cache
           persistentVolumeClaim:
-            claimName: jellyfin-cache-pvc
+            claimName: jellyfin-cache-backup
         - name: yarr
           nfs:
             server: robin.home.nest

services/jellyfin/jellyfin-pvcs.yaml

@@ -16,7 +16,7 @@ spec:
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: jellyfin-cache-pvc
+  name: jellyfin-cache-backup
   labels:
     recurring-job.longhorn.io/source: "enabled"
     recurring-job-group.longhorn.io/app-config-group: "enabled"