Kubernetes日志收集 - 云原生基站

1 安装loki#

1.1 更新配置文件#

1.1.1 配置集群参数#

1
  memberlist:
2
    join_members: ["loki-1", "loki-2", "loki-3"]
3
    dead_node_reclaim_time: 30s
4
    gossip_to_dead_nodes_time: 15s
5
    left_ingesters_timeout: 30s
6
    bind_addr: ['0.0.0.0']
7
    bind_port: 7946
8
  ingester:
9
    chunk_idle_period: 3m
10
    chunk_block_size: 262144
11
    chunk_retain_period: 1m
12
    max_transfer_retries: 0
13
    lifecycler:
14
      join_after: 60s
15
      observe_period: 5s
16
      ring:
17
        kvstore:
18
          store: memberlist
19
        replication_factor: 2

1.1.2 配置持久化#

1
persistence:
2
  enabled: true
3
  accessModes:
4
  - ReadWriteOnce
5
  size: 300Gi
6
  annotations: {}

helm upgrade --install loki --namespace=loki . --set limitsConfig.ingestionBurstSizeMb=40 --set limitsConfig.ingestionRateMb=30 --set fluent-bit.enabled=false,promtail.enabled=true,grafana.enabled=true,prometheus.enabled=true,prometheus.alertmanager.persistentVolume.enabled=true,prometheus.server.persistentVolume.enabled=true

注意这里用promtail，promtail.enabled=true

我这里是测试环境，就没配置那么多参数，建议下载下来自定义values.yaml

2 安装promtail#

helm pull grafana/promtail 查看loki的svc信息

解压编辑values.yaml

1，配置loki地址

lokiAddress: http://loki-headless:3100/api/v1/push

主要添加监控网关的日志

2，只收集业务命名空间(dev与test的日志，一定要放最后

1
          - source_labels: [__meta_kubernetes_namespace]
2
            action: keep
3
            regex: test|dev

3，默认的标签太多，目前用不到，因为只看业务日志

1
    pipelineStages:
2
      - cri: {}
3
    common:
4
      - action: replace
5
        source_labels:
6
          - __meta_kubernetes_pod_node_name
7
        target_label: node_name
8
      - action: replace
9
        source_labels:
10
          - __meta_kubernetes_namespace
11
        target_label: namespace
12
      - action: replace
13
        source_labels:
14
          - __meta_kubernetes_pod_name
15
        target_label: pod
16
      - action: replace
17
        replacement: /var/log/pods/*$1/*.log
18
        separator: /
19
        source_labels:
20
          - __meta_kubernetes_pod_uid
21
          - __meta_kubernetes_pod_container_name
22
        target_label: __path__
23
      - action: replace
24
        replacement: /var/log/pods/*$1/*.log
25
        regex: true/(.*)
26
        separator: /
27
        source_labels:
28
          - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
29
          - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
30
          - __meta_kubernetes_pod_container_name
31
        target_label: __path__

1
    scrapeConfigs: |
2
      - job_name: kubernetes-pods
3
        pipeline_stages:
4
          {{- toYaml .Values.config.snippets.pipelineStages | nindent 4 }}
5
        kubernetes_sd_configs:
6
          - role: pod
7
        relabel_configs:
8
          - source_labels: [__meta_kubernetes_pod_label_app]
9
            action: drop
10
            regex: finance-gateway
11
          - source_labels:
12
              - __meta_kubernetes_pod_controller_name
13
            regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
14
            action: replace
15
            target_label: __tmp_controller_name
16
          - source_labels:
17
              - __meta_kubernetes_pod_label_app_kubernetes_io_name
18
              - __meta_kubernetes_pod_label_app
19
              - __tmp_controller_name
20
              - __meta_kubernetes_pod_name
21
            regex: ^;*([^;]+)(;.*)?$
22
            action: replace
23
            target_label: app
24
          - source_labels:
25
              - __meta_kubernetes_pod_label_app_kubernetes_io_component
26
              - __meta_kubernetes_pod_label_component
27
            regex: ^;*([^;]+)(;.*)?$
28
            action: replace
29
            target_label: component
30
          - source_labels: [__meta_kubernetes_namespace]
31
            action: keep
32
            regex: test|dev
33
          {{- if .Values.config.snippets.addScrapeJobLabel }}
34
          - replacement: kubernetes-pods
35
            target_label: scrape_job
36
          {{- end }}
37
          {{- toYaml .Values.config.snippets.common | nindent 4 }}
38
          {{- with .Values.config.snippets.extraRelabelConfigs }}
39
          {{- toYaml . | nindent 4 }}
40
          {{- end }}

4，因为上面挂载网关的日志了，这里不收集网关的pod日志

1
          - source_labels: [__meta_kubernetes_pod_label_app]
2
            action: drop
3
            regex: finance-gateway

5，新加个收集gateway任务，注释部分是隐藏ip地址最后一位，不需要所以注释了

1
      - job_name: access
2
      #  pipeline_stages:
3
      #  - replace:
4
      #      expression: '(?:[0-9]{1,3}\.){3}([0-9]{1,3})'
5
      #      replace: '***'
6
        static_configs:
7
          - targets:
8
              - localhost
9
            labels:
10
              job: nginx_access_log
11
              host: appfelstrudel
12
              agent: promtail
13
              __path__: /var/log/access/*.log
14
          - targets:
15
              - localhost
16
            labels:
17
              job: xiaobaihua
18
              __path__: /var/log/access/xiaobaihua-gateway.access.log
19
          - targets:
20
              - localhost
21
            labels:
22
              job: reapal
23
              __path__: /var/log/access/reapal-gateway.access.log

6，部署promtail

helm install promtail -n loki .

3 配置grafana#

3.1 添加Loki地址#

3.2 查看标签#