diff --git a/monitoring/stack/monitoring-external-secrets.yaml b/monitoring/stack/monitoring-external-secrets.yaml new file mode 100644 index 0000000..b82d77a --- /dev/null +++ b/monitoring/stack/monitoring-external-secrets.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: grafana-admin-credentials + namespace: monitoring +spec: + refreshInterval: "1h" + secretStoreRef: + name: vault-external-secrets-store + kind: ClusterSecretStore + target: + name: grafana-admin-credentials + creationPolicy: Owner + data: + - secretKey: admin-user + remoteRef: + key: grafana + property: username + - secretKey: admin-password + remoteRef: + key: grafana + property: password diff --git a/monitoring/stack/monitoring-services.yaml b/monitoring/stack/monitoring-services.yaml new file mode 100644 index 0000000..a036ec9 --- /dev/null +++ b/monitoring/stack/monitoring-services.yaml @@ -0,0 +1,41 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: monitoring-ingress + namespace: monitoring + annotations: + # Routes HTTP traffic through Traefik's default web entrypoint + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + rules: + - host: grafana.k3s.home.nest + http: + paths: + - path: / + pathType: Prefix + backend: + service: + # Point directly to the native Helm-managed service + name: kube-prometheus-stack-grafana + port: + number: 80 # The built-in chart service listens on port 80 + - host: prometheus.k3s.home.nest + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kube-prometheus-stack-prometheus + port: + number: 9090 + - host: alertmanager.k3s.home.nest + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kube-prometheus-stack-alertmanager + port: + number: 9093 diff --git a/monitoring/stack/monitoring-stack.yaml b/monitoring/stack/monitoring-stack.yaml index f89828c..2847911 100644 --- a/monitoring/stack/monitoring-stack.yaml +++ b/monitoring/stack/monitoring-stack.yaml @@ -2,7 +2,7 @@ apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: name: kube-prometheus-stack - namespace: kube-system # The controller looks here, but we will deploy to 'monitoring' + namespace: kube-system spec: chart: kube-prometheus-stack repo: https://prometheus-community.github.io/helm-charts @@ -23,14 +23,17 @@ spec: externalUrl: http://prometheus.k3s.home.nest storageSpec: volumeClaimTemplate: + metadata: + # Automatically opt this volume into your backup/snapshot rules + labels: + recurring-job.longhorn.io/source: "enabled" + recurring-job-group.longhorn.io/app-config-group: "enabled" spec: - storageClassName: "" + storageClassName: "longhorn" # Switch to longhorn dynamic storage + accessModes: ["ReadWriteOnce"] resources: requests: storage: 20Gi - selector: - matchLabels: - app: prometheus-data alertmanager: config: @@ -41,12 +44,11 @@ spec: group_wait: 30s group_interval: 5m repeat_interval: 12h - receiver: 'discord-notifications' # Point the default route here + receiver: 'discord-notifications' routes: - matchers: - alertname = Watchdog receiver: 'null' - # Inhibition rules reduce noise by silencing related alerts inhibit_rules: - source_matchers: [severity="critical"] target_matchers: [severity="warning"] @@ -57,7 +59,6 @@ spec: discord_configs: - webhook_url: 'https://discord.com/api/webhooks/1483240068197126300/zI_k1nEc8U8Fd0hiJc7gBG0TjklblmYrSDvpbTMiposGc32_0UREpLjch1FggFvluV3D' send_resolved: true - # Formatting the message for better readability title: '{{ .Status | toUpper }}: {{ .GroupLabels.alertname }}' message: |- **Summary:** {{ range .Alerts }}{{ .Annotations.summary }}{{ end }} @@ -67,14 +68,16 @@ spec: externalUrl: http://alertmanager.k3s.home.nest storage: volumeClaimTemplate: + metadata: + labels: + recurring-job.longhorn.io/source: "enabled" + recurring-job-group.longhorn.io/app-config-group: "enabled" spec: - storageClassName: "" + storageClassName: "longhorn" # Switch to longhorn dynamic storage + accessModes: ["ReadWriteOnce"] resources: requests: storage: 2Gi - selector: - matchLabels: - app: alertmanager-data grafana: admin: @@ -95,9 +98,8 @@ spec: persistence: enabled: true type: pvc - storageClassName: grafana-nfs + storageClassName: "longhorn" # Switch to longhorn dynamic storage size: 5Gi - finalizers: - - kubernetes.io/pvc-protection - selectorLabels: - app: grafana-data + labels: + recurring-job.longhorn.io/source: "enabled" + recurring-job-group.longhorn.io/app-config-group: "enabled" diff --git a/monitoring/stack/nfs-claims.yaml b/monitoring/stack/nfs-claims.yaml deleted file mode 100644 index 5b8cd68..0000000 --- a/monitoring/stack/nfs-claims.yaml +++ /dev/null @@ -1,55 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: prometheus-nfs-pv - labels: - app: prometheus-data -spec: - capacity: - storage: 20Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - nfs: - server: birdfeeder2.home.nest - path: /k3s/prometheus - mountOptions: - - nfsvers=4.1 ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: alertmanager-nfs-pv - labels: - app: alertmanager-data -spec: - capacity: - storage: 2Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - nfs: - server: birdfeeder2.home.nest - path: /k3s/alertmanager - mountOptions: - - nfsvers=4.1 ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: grafana-nfs-pv - labels: - app: grafana-data -spec: - storageClassName: grafana-nfs - capacity: - storage: 5Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - nfs: - server: birdfeeder2.home.nest - path: /k3s/grafana - mountOptions: - - nfsvers=4.1 diff --git a/monitoring/stack/services-external.yaml b/monitoring/stack/services-external.yaml deleted file mode 100644 index e325284..0000000 --- a/monitoring/stack/services-external.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana-internal - namespace: monitoring -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: grafana - ports: - - name: http - port: 3000 - targetPort: 3000 ---- -apiVersion: v1 -kind: Service -metadata: - name: prometheus-internal - namespace: monitoring -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: prometheus - operator.prometheus.io/name: kube-prometheus-stack-prometheus - ports: - - name: http - port: 9090 - targetPort: 9090 ---- -apiVersion: v1 -kind: Service -metadata: - name: alertmanager-internal - namespace: monitoring -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: alertmanager - app.kubernetes.io/instance: kube-prometheus-stack-alertmanager - ports: - - name: http - port: 9093 - targetPort: 9093 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: monitoring-ingress - namespace: monitoring - annotations: - traefik.ingress.kubernetes.io/router.entrypoints: web -spec: - rules: - - host: grafana.k3s.home.nest - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: grafana-internal - port: - number: 3000 - - host: prometheus.k3s.home.nest - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: prometheus-internal - port: - number: 9090 - - host: alertmanager.k3s.home.nest - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: alertmanager-internal - port: - number: 9093