k3s/monitoring/monitoring-stack.yaml

apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
  name: kube-prometheus-stack
  namespace: kube-system
spec:
  chart: kube-prometheus-stack
  repo: https://prometheus-community.github.io/helm-charts
  targetNamespace: monitoring
  createNamespace: true
  valuesContent: |-
    kubeControllerManager:
      enabled: false
    kubeScheduler:
      enabled: false
    kubeProxy:
      enabled: false

    # Filter out IPV6 monitors
    kubelet:
      enabled: true
      serviceMonitor:
        cAdvisorRelabelings:
          - sourceLabels: [__address__]
            regex: '^\[.*'
            action: drop
        probesRelabelings:
          - sourceLabels: [__address__]
            regex: '^\[.*'
            action: drop
        relabelings:
          - sourceLabels: [__address__]
            regex: '^\[.*'
            action: drop

    prometheus:
      prometheusSpec:
        serviceMonitorSelectorNilUsesHelmValues: false
        retention: 10d
        externalUrl: http://prometheus.k3s.home.nest
        storageSpec:
          volumeClaimTemplate:
            metadata:
              # Automatically opt this volume into your backup/snapshot rules
              labels:
                recurring-job.longhorn.io/source: "enabled"
                recurring-job-group.longhorn.io/app-config-group: "enabled"
            spec:
              storageClassName: "longhorn" # Switch to longhorn dynamic storage
              accessModes: ["ReadWriteOnce"]
              resources:
                requests:
                  storage: 20Gi

    alertmanager:
      config:
        global:
          resolve_timeout: 5m
        route:
          group_by: ['alertname', 'cluster', 'service']
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 12h 
          receiver: 'matrix-notifications'
          routes:
            - matchers:
                - alertname = Watchdog
              receiver: 'null'
        inhibit_rules:
          - source_matchers: [severity="critical"]
            target_matchers: [severity="warning"]
            equal: ['alertname', 'dev', 'instance']
        receivers:
          - name: 'null'
          - name: 'matrix-notifications'
            webhook_configs:
              - url_file: '/etc/alertmanager/secrets/matrix-alertmanager-secret/WEBHOOK_URL'
                send_resolved: true
      alertmanagerSpec:
        secrets:
          - matrix-alertmanager-secret
        externalUrl: http://alertmanager.k3s.home.nest
        storage:
          volumeClaimTemplate:
            metadata:
              labels:
                recurring-job.longhorn.io/source: "enabled"
                recurring-job-group.longhorn.io/app-config-group: "enabled"
            spec:
              storageClassName: "longhorn" # Switch to longhorn dynamic storage
              accessModes: ["ReadWriteOnce"]
              resources:
                requests:
                  storage: 2Gi

    grafana:
      admin:
        existingSecret: "grafana-admin-credentials"
        userKey: admin-user
        passwordKey: admin-password
      podSecurityContext:
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
      initChownData:
        enabled: false
      grafana.ini:
        server:
          domain: grafana.k3s.home.nest
          root_url: http://grafana.k3s.home.nest
          serve_from_sub_path: false
      persistence:
        enabled: true
        type: pvc
        storageClassName: "longhorn" # Switch to longhorn dynamic storage
        size: 5Gi
        labels:
          recurring-job.longhorn.io/source: "enabled"
          recurring-job-group.longhorn.io/app-config-group: "enabled"