1
- job_name: prb-haproxy
2
  honor_timestamps: true
3
  scrape_interval: 2m
4
  scrape_timeout: 10s
5
  metrics_path: /metrics
6
  scheme: http
7
  follow_redirects: true
8
  static_configs:
9
  - targets:
10
    - 10.0.0.1:1080
11
    - 10.0.0.2:1080
12
    labels:
13
      cluster: gitlab
14
      haproxy: yes
15
- job_name: prb-keepalived
16
  honor_timestamps: true
17
  scrape_interval: 2m
18
  scrape_timeout: 10s
19
  metrics_path: /metrics
20
  scheme: http
21
  follow_redirects: true
22
  static_configs:
23
  - targets:
24
    - 10.0.0.1:9713
25
    - 10.0.0.2:9713
26
    labels:
27
      cluster: gitlab
28
      haproxy: yes
29
- job_name: prb-node
30
  honor_timestamps: true
31
  scrape_interval: 2m
32
  scrape_timeout: 10s
33
  metrics_path: /metrics
34
  scheme: http
35
  follow_redirects: true
36
  static_configs:
37
  - targets:
38
    - 10.0.0.1:9100
39
    - 10.0.0.2:9100
40
    labels:
41
      cluster: gitlab
42
      haproxynode: "yes"

1
groups:
2
- name: HAProxy.rules
3
  rules:
4
  - alert: HAProxy down
5
    expr: up{haproxy="yes"}==0
6
    for: 1m
7
    labels:
8
      severity: critical
9
    annotations:
10
      description: 'HAProxy down (instance {{ $labels.instance }})'
11
      summary: HAProxy down
12
  - alert: HAProxy server down
13
    expr: haproxy_server_status{state="UP"}==0
14
    for: 1m
15
    labels:
16
      severity: critical
17
    annotations:
18
      description: 'HAProxy server is down\n SERVER = {{ $labels.server }}'
19
      summary: HAProxy server down (instance {{ $labels.instance }})
20
  - alert: HAProxy frontend down
21
    expr: haproxy_frontend_status{state="UP"}==0
22
    for: 1m
23
    labels:
24
      severity: critical
25
    annotations:
26
      description: 'HAProxy frontend is down\n PROXY = {{ $labels.proxy }}'
27
      summary: HAProxy frontend down (instance {{ $labels.instance }})
28
  - alert: HAProxy backend down
29
    expr: haproxy_backend_status{state="UP"}==0
30
    for: 1m
31
    labels:
32
      severity: critical
33
    annotations:
34
      summary: HAProxy backend down (instance {{ $labels.instance }})
35
      description: 'HAProxy backend is down\n PROXY = {{ $labels.proxy }}'
36
  - alert: HAProxy backend max active session
37
    expr: ((sum by (proxy) (avg_over_time(haproxy_backend_max_sessions[5m]) * 100) / sum by (proxy) (avg_over_time(haproxy_backend_limit_sessions[5m])))) > 80
38
    for: 5m
39
    labels:
40
      severity: warning
41
    annotations:
42
      summary: HAProxy backend max active session (instance {{ $labels.instance }})
43
      description: 'HAproxy backend {{ $labels.proxy }} is reaching session limit (> 80%).\n  VALUE = {{ $value }}'
44
  - alert: HAProxy pending requests
45
    expr: sum by (proxy) (haproxy_backend_current_queue) > 0
46
    for: 1m
47
    labels:
48
      severity: warning
49
    annotations:
50
      summary: HAProxy pending requests (instance {{ $labels.instance }})
51
      description: 'Some HAProxy requests are pending on {{ $labels.proxy }}\n  VALUE = {{ $value }}'
52
  - alert: HAProxy server healthcheck failure
53
    expr: increase(haproxy_server_check_failures_total[5m]) > 0
54
    for: 5m
55
    labels:
56
      severity: warning
57
    annotations:
58
      summary: HAProxy server healthcheck failure (instance {{ $labels.instance }})
59
      description: 'Some server healthcheck are failing on {{ $labels.server }}'
60
  - alert: HAProxy frontend security blocked requests
61
    expr: sum by (proxy) (rate(haproxy_frontend_requests_denied_total[5m])) > 10
62
    for: 5m
63
    labels:
64
      severity: warning
65
    annotations:
66
      summary: HAProxy frontend security blocked requests (instance {{ $labels.instance }})
67
      description: 'HAProxy is blocking requests for security reason\n PROXY = {{ $labels.proxy }}\n VALUE = {{ $value }}'
68
  - alert: HAProxy server connection errors
69
    expr: sum by (proxy,server) (rate(haproxy_server_connection_errors_total[5m])) > 100
70
    for: 5m
71
    labels:
72
      severity: critical
73
    annotations:
74
      summary: HAProxy server connection errors (instance {{ $labels.instance }})
75
      description: 'Too many connection errors to {{ $labels.server }} server (> 100 req/s). Request throughput may be too high.\n  VALUE = {{ $value }}'
76
  - alert: HAProxy backend connection errors
77
    expr: sum by (proxy) (rate(haproxy_backend_connection_errors_total[5m])) > 100
78
    for: 5m
79
    labels:
80
      severity: critical
81
    annotations:
82
      summary: HAProxy backend connection errors (instance {{ $labels.instance }})
83
      description: 'Too many connection errors to {{ $labels.proxy }} (> 100 req/s). Request throughput may be too high.\n  VALUE = {{ $value }}'
84
  - alert: HAProxy retry high
85
    expr: sum by (proxy) (rate(haproxy_backend_retry_warnings_total[5m]))  > 10
86
    for: 5m
87
    labels:
88
      severity: warning
89
    annotations:
90
      summary: HAProxy retry high (instance {{ $labels.instance }})
91
      description: 'High rate of retry on {{ $labels.proxy }}\n  VALUE = {{ $value }}'
92
  - alert: HAProxy high HTTP 5xx error rate backend
93
    expr: sum by (proxy) (rate(haproxy_backend_http_responses_total{code="5xx"}[5m])) / sum by (proxy) (rate(haproxy_backend_http_responses_total[5m])) > 5
94
    for: 5m
95
    labels:
96
      severity: critical
97
    annotations:
98
      summary: HAProxy high HTTP 5xx error rate backend (instance {{ $labels.instance }})
99
      description: 'Too many HTTP requests with status 5xx (> 5%) on backend {{ $labels.proxy }}\n  VALUE = {{ $value }}'
100
  - alert: HAProxy high HTTP 4xx error rate backend
101
    expr: sum by (proxy) (rate(haproxy_backend_http_responses_total{code="4xx"}[5m])) / sum by (proxy) (rate(haproxy_backend_http_responses_total[5m])) > 5
102
    for: 5m
103
    labels:
104
      severity: critical
105
    annotations:
106
      summary: HAProxy high HTTP 4xx error rate backend (instance {{ $labels.instance }})
107
      description: 'Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.proxy }}\n  VALUE = {{ $value }}'
108
  - alert: HAProxy HTTP slowing down
109
    expr: avg by (proxy) (haproxy_backend_total_time_average_seconds) > 1
110
    for: 1m
111
    labels:
112
      severity: warning
113
    annotations:
114
      summary: HAProxy HTTP slowing down (instance {{ $labels.instance }})
115
      description: 'Average request time is increasing\n PROXY = {{ $labels.proxy }}\n VALUE = {{ $value }}'