Migrate monitoring stack to longhorn

monitoring/stack/monitoring-external-secrets.yaml

@@ -0,0 +1,22 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: grafana-admin-credentials
+  namespace: monitoring
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    name: vault-external-secrets-store
+    kind: ClusterSecretStore
+  target:
+    name: grafana-admin-credentials
+    creationPolicy: Owner
+  data:
+  - secretKey: admin-user
+    remoteRef:
+      key: grafana
+      property: username
+  - secretKey: admin-password
+    remoteRef:
+      key: grafana
+      property: password

monitoring/stack/monitoring-services.yaml

@@ -0,0 +1,41 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: monitoring-ingress
+  namespace: monitoring
+  annotations:
+    # Routes HTTP traffic through Traefik's default web entrypoint
+    traefik.ingress.kubernetes.io/router.entrypoints: web
+spec:
+  rules:
+  - host: grafana.k3s.home.nest
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            # Point directly to the native Helm-managed service
+            name: kube-prometheus-stack-grafana
+            port:
+              number: 80 # The built-in chart service listens on port 80
+  - host: prometheus.k3s.home.nest
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: kube-prometheus-stack-prometheus
+            port:
+              number: 9090
+  - host: alertmanager.k3s.home.nest
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: kube-prometheus-stack-alertmanager
+            port:
+              number: 9093

monitoring/stack/monitoring-stack.yaml

@@ -2,7 +2,7 @@ apiVersion: helm.cattle.io/v1
 kind: HelmChart
 metadata:
   name: kube-prometheus-stack
-  namespace: kube-system # The controller looks here, but we will deploy to 'monitoring'
+  namespace: kube-system
 spec:
   chart: kube-prometheus-stack
   repo: https://prometheus-community.github.io/helm-charts
@@ -23,14 +23,17 @@ spec:
         externalUrl: http://prometheus.k3s.home.nest
         storageSpec:
           volumeClaimTemplate:
+            metadata:
+              # Automatically opt this volume into your backup/snapshot rules
+              labels:
+                recurring-job.longhorn.io/source: "enabled"
+                recurring-job-group.longhorn.io/app-config-group: "enabled"
             spec:
-              storageClassName: ""
+              storageClassName: "longhorn" # Switch to longhorn dynamic storage
+              accessModes: ["ReadWriteOnce"]
               resources:
                 requests:
                   storage: 20Gi
-              selector:
-                matchLabels:
-                  app: prometheus-data
 
     alertmanager:
       config:
@@ -41,12 +44,11 @@ spec:
           group_wait: 30s
           group_interval: 5m
           repeat_interval: 12h 
-          receiver: 'discord-notifications' # Point the default route here
+          receiver: 'discord-notifications'
           routes:
             - matchers:
                 - alertname = Watchdog
               receiver: 'null'
-        # Inhibition rules reduce noise by silencing related alerts
         inhibit_rules:
           - source_matchers: [severity="critical"]
             target_matchers: [severity="warning"]
@@ -57,7 +59,6 @@ spec:
             discord_configs:
               - webhook_url: 'https://discord.com/api/webhooks/1483240068197126300/zI_k1nEc8U8Fd0hiJc7gBG0TjklblmYrSDvpbTMiposGc32_0UREpLjch1FggFvluV3D'
                 send_resolved: true
-                # Formatting the message for better readability
                 title: '{{ .Status | toUpper }}: {{ .GroupLabels.alertname }}'
                 message: |-
                   **Summary:** {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
@@ -67,14 +68,16 @@ spec:
         externalUrl: http://alertmanager.k3s.home.nest
         storage:
           volumeClaimTemplate:
+            metadata:
+              labels:
+                recurring-job.longhorn.io/source: "enabled"
+                recurring-job-group.longhorn.io/app-config-group: "enabled"
             spec:
-              storageClassName: ""
+              storageClassName: "longhorn" # Switch to longhorn dynamic storage
+              accessModes: ["ReadWriteOnce"]
               resources:
                 requests:
                   storage: 2Gi
-              selector:
-                matchLabels:
-                  app: alertmanager-data
 
     grafana:
       admin:
@@ -95,9 +98,8 @@ spec:
       persistence:
         enabled: true
         type: pvc
-        storageClassName: grafana-nfs
+        storageClassName: "longhorn" # Switch to longhorn dynamic storage
         size: 5Gi
-        finalizers:
+        labels:
-          - kubernetes.io/pvc-protection
+          recurring-job.longhorn.io/source: "enabled"
-        selectorLabels:
+          recurring-job-group.longhorn.io/app-config-group: "enabled"
-          app: grafana-data

monitoring/stack/nfs-claims.yaml

@@ -1,55 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: prometheus-nfs-pv
-  labels:
-    app: prometheus-data
-spec:
-  capacity:
-    storage: 20Gi
-  accessModes:
-    - ReadWriteOnce
-  persistentVolumeReclaimPolicy: Retain
-  nfs:
-    server: birdfeeder2.home.nest
-    path: /k3s/prometheus
-  mountOptions:
-    - nfsvers=4.1
----
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: alertmanager-nfs-pv
-  labels:
-    app: alertmanager-data
-spec:
-  capacity:
-    storage: 2Gi
-  accessModes:
-    - ReadWriteOnce
-  persistentVolumeReclaimPolicy: Retain
-  nfs:
-    server: birdfeeder2.home.nest
-    path: /k3s/alertmanager
-  mountOptions:
-    - nfsvers=4.1
----
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: grafana-nfs-pv
-  labels:
-    app: grafana-data
-spec:
-  storageClassName: grafana-nfs
-  capacity:
-    storage: 5Gi
-  accessModes:
-    - ReadWriteOnce
-  persistentVolumeReclaimPolicy: Retain
-  nfs:
-    server: birdfeeder2.home.nest
-    path: /k3s/grafana
-  mountOptions:
-    - nfsvers=4.1

monitoring/stack/services-external.yaml

@@ -1,84 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: grafana-internal
-  namespace: monitoring
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: grafana
-  ports:
-    - name: http
-      port: 3000
-      targetPort: 3000
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: prometheus-internal
-  namespace: monitoring
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: prometheus
-    operator.prometheus.io/name: kube-prometheus-stack-prometheus
-  ports:
-    - name: http
-      port: 9090
-      targetPort: 9090
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: alertmanager-internal
-  namespace: monitoring
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: alertmanager
-    app.kubernetes.io/instance: kube-prometheus-stack-alertmanager
-  ports:
-    - name: http
-      port: 9093
-      targetPort: 9093
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: monitoring-ingress
-  namespace: monitoring
-  annotations:
-    traefik.ingress.kubernetes.io/router.entrypoints: web
-spec:
-  rules:
-  - host: grafana.k3s.home.nest
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: grafana-internal
-            port:
-              number: 3000
-  - host: prometheus.k3s.home.nest
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: prometheus-internal
-            port:
-              number: 9090
-  - host: alertmanager.k3s.home.nest
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: alertmanager-internal
-            port:
-              number: 9093