feat: enables prometheus rules and alerts (#292)

* feat: enables prometheus rules and alerts which can be sent to alertmanager

* chore: adding configs for alertmanager, alert, and prom

* chore: alerts WIP

* chore: alerts WIP

* chore: alerts WIP

* chore: setRules API will update rules

* chore: initialization of prometheus related stuff moved to separate function

* chore: alerts WIP

* chore: alerts WIP

* fix: r.promConfig was nil

* feat: routing alertmanager apis to alertmanager service at nginx

* chore: not writing to localDB if string parsing gives error

* feat: list alerts API

* chore: error in creating multiple groups

* feat: CRUD APIs for rules working

* chore: changed prometheus version

* chore: updated AlertingRuleResponse struct's Id json value

* chore: updated prometheus's version

* chore: will load rules from database on bootup

* feat: crud APIs for notification channels WIP

* fix: changed ALERTMANAGER_API_PREFIX

* chore: enabling scrape and notify discover manager

* chore: fixing path for signoz.db

* chore: used transactions for rules APIs

* chore: editchannel API updated and other apis refactored

* chore: fixed merge conflicts

* chore: changing createChannel api from yaml to json reader

* chore: changing editChannel api from yaml to json reader

* chore: porting loadChannels to json format

* chore: editRule returning rule not found

* chore: pre-release

* chore: fixed db path for persistence

* release: v0.5.0

Author: ankitnayan

deploy/docker/clickhouse-setup/alertmanager.yml

@@ -0,0 +1,35 @@
+global:
+  resolve_timeout: 1m
+  slack_api_url: 'https://hooks.slack.com/services/xxx'
+
+route:
+  receiver: 'slack-notifications'
+
+receivers:
+- name: 'slack-notifications'
+  slack_configs:
+  - channel: '#alerts'
+    send_resolved: true
+    icon_url: https://avatars3.githubusercontent.com/u/3380462
+    title: |-
+     [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}
+    text: >-
+     {{ range .Alerts -}}
+     *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
+
+     *Description:* {{ .Annotations.description }}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
+       {{ end }}
+     {{ end }}

deploy/docker/clickhouse-setup/alerts.yml

@@ -0,0 +1,11 @@
+groups:
+- name: ExampleCPULoadGroup
+  rules:
+  - alert: HighCpuLoad
+    expr: system_cpu_load_average_1m > 0.1
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: High CPU load
+      description: "CPU load is > 0.1\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -21,8 +21,19 @@ services:
         timeout: 5s
         retries: 3
 
+  alertmanager:
+    image: signoz/alertmanager:0.5.0
+    volumes:
+      - ./alertmanager.yml:/prometheus/alertmanager.yml
+      - ./data/alertmanager:/data
+    command:
+      - '--config.file=/prometheus/alertmanager.yml'
+      - '--storage.path=/data'
+    ports:
+      - 9093:9093
+
   query-service:
-    image: signoz/query-service:0.4.5
+    image: signoz/query-service:0.5.0
     container_name: query-service
     command: ["-config=/root/config/prometheus.yml"]
     ports:
@@ -43,7 +54,7 @@ services:
         condition: service_healthy
 
   frontend:
-    image: signoz/frontend:0.4.5
+    image: signoz/frontend:0.5.0
     container_name: frontend
 
     depends_on:

deploy/docker/clickhouse-setup/prometheus.yml

@@ -9,12 +9,13 @@ alerting:
   alertmanagers:
   - static_configs:
     - targets:
-      # - alertmanager:9093
+      - alertmanager:9093
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
   # - "first_rules.yml"
   # - "second_rules.yml"
+  - 'alerts.yml'
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.

deploy/docker/common/nginx-config.conf

@@ -16,6 +16,9 @@ server {
     index  index.html index.htm;
     try_files $uri $uri/ /index.html;
     }
+    location /api/alertmanager{
+    proxy_pass http://alertmanager:9093/api/v2;
+    }
     location /api {
     proxy_pass http://query-service:8080/api;