Monitoring

helmchartconfig/traefik/service-traefik.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: traefik-metrics
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: traefik
+    metrics: enabled
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: traefik
+  ports:
+    - name: metrics
+      port: 9100
+      targetPort: 9100

helmchartconfig/traefik/traefik-config.yaml

@@ -1,4 +1,3 @@
----
 apiVersion: helm.cattle.io/v1
 kind: HelmChartConfig
 metadata:
@@ -7,6 +6,7 @@ metadata:
 spec:
   valuesContent: |-
     additionalArguments:
+      - "--entryPoints.metrics.address=:9100"
       - "--metrics.prometheus=true"
       - "--metrics.prometheus.entryPoint=metrics"
     ports:

monitoring/servicemonitors/traefik-monitor.yaml

@@ -1,4 +1,3 @@
----
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -7,10 +6,9 @@ metadata:
   labels:
     release: kube-prometheus-stack
 spec:
-  jobLabel: traefik
   selector:
     matchLabels:
-      app.kubernetes.io/name: traefik
+      metrics: enabled # This matches the new Service we just made
   namespaceSelector:
     matchNames:
       - kube-system
@@ -18,3 +16,6 @@ spec:
     - port: metrics
       interval: 30s
       path: /metrics
+      relabelings:
+        - targetLabel: job
+          replacement: traefik

monitoring/stack/monitoring-stack.yaml

@@ -9,6 +9,13 @@ spec:
   targetNamespace: monitoring
   createNamespace: true
   valuesContent: |-
+    kubeControllerManager:
+      enabled: false
+    kubeScheduler:
+      enabled: false
+    kubeProxy:
+      enabled: false
+
     prometheus:
       prometheusSpec:
         serviceMonitorSelectorNilUsesHelmValues: false
@@ -26,6 +33,36 @@ spec:
                   app: prometheus-data
 
     alertmanager:
+      config:
+        global:
+          resolve_timeout: 5m
+        route:
+          group_by: ['alertname', 'cluster', 'service']
+          group_wait: 30s
+          group_interval: 5m
+          repeat_interval: 12h 
+          receiver: 'discord-notifications' # Point the default route here
+          routes:
+            - matchers:
+                - alertname = Watchdog
+              receiver: 'null'
+        # Inhibition rules reduce noise by silencing related alerts
+        inhibit_rules:
+          - source_matchers: [severity="critical"]
+            target_matchers: [severity="warning"]
+            equal: ['alertname', 'dev', 'instance']
+        receivers:
+          - name: 'null'
+          - name: 'discord-notifications'
+            discord_configs:
+              - webhook_url: 'https://discord.com/api/webhooks/1483240068197126300/zI_k1nEc8U8Fd0hiJc7gBG0TjklblmYrSDvpbTMiposGc32_0UREpLjch1FggFvluV3D'
+                send_resolved: true
+                # Formatting the message for better readability
+                title: '{{ .Status | toUpper }}: {{ .GroupLabels.alertname }}'
+                message: |-
+                  **Summary:** {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
+                  **Description:** {{ range .Alerts }}{{ .Annotations.description }}{{ end }}
+                  **Severity:** {{ range .Alerts }}{{ .Labels.severity }}{{ end }}
       alertmanagerSpec:
         externalUrl: http://alertmanager.home.nest
         storage: