通过 Ngnix 日志监控

1
yum -y install yum-utils
2
vim /etc/yum.repos.d/nginx.repo
3
[nginx-stable]
4
name=nginx stable repo
5
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
6
gpgcheck=1
7
enabled=1
8
gpgkey=https://nginx.org/keys/nginx_signing.key
9
module_hotfixes=true
10

11
[nginx-mainline]
12
name=nginx mainline repo
13
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
14
gpgcheck=1
15
enabled=0
16
gpgkey=https://nginx.org/keys/nginx_signing.key
17
module_hotfixes=true
18
yum -y install nginx

1
vim /etc/nginx/nginx.conf
2
    map "$time_iso8601 # $msec" $time_iso8601_ms { "~(^[^+]+)(\+[0-9:]+) # \d+\.(\d+)$" $1.$3$2; }
3
    log_format main
4
        '{"timestamp":"$time_iso8601_ms",'
5
        '"server_ip":"$server_addr",'
6
        '"remote_ip":"$remote_addr",'
7
        '"xff":"$http_x_forwarded_for",'
8
        '"remote_user":"$remote_user",'
9
        '"domain":"$host",'
10
        '"url":"$request_uri",'
11
        '"referer":"$http_referer",'
12
        '"upstreamtime":"$upstream_response_time",'
13
        '"responsetime":"$request_time",'
14
        '"request_method":"$request_method",'
15
        '"status":"$status",'
16
        '"response_length":"$bytes_sent",'
17
        '"request_length":"$request_length",'
18
        '"protocol":"$server_protocol",'
19
        '"upstreamhost":"$upstream_addr",'
20
        '"http_user_agent":"$http_user_agent"'
21
        '}';

1
cat >> docker-compose.yml <<EOF
2
services:
3
  clickhouse:
4
    image: clickhouse:25.6
5
    container_name: clickhouse
6
    network_mode: host
7
    restart: always
8
    expose:
9
      - "8123"
10
      - "9000"
11
    volumes:
12
      #- ./config/docker_related_config.xml:/etc/clickhouse-server/config.d/docker_related_config.xml:rw
13
      #- ./config/config.xml:/etc/clickhouse-server/config.xml:rw
14
      #- ./config/users.xml:/etc/clickhouse-server/users.xml:rw
15
      - /etc/localtime:/etc/localtime:ro
16
      - ./log:/var/log/clickhouse-server
17
      - ./data:/var/lib/clickhouse/:rw
18
EOF
19
mkdir config
20
docker cp clickhouse:/etc/clickhouse-server/config.xml ./config
21
docker cp clickhouse:/etc/clickhouse-server/users.xml ./config
22
docker cp clickhouse:/etc/clickhouse-server/config.d/docker_related_config.xml ./config

1
#PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
2
#9aiHCto/
3
#885551fc11e6ce714c2a947f0fb958945c84804c1cfcff575b67ea795ef357ef
4
#方便查看我没有加密,如果需要加密<password>1qaz@WSX</password>修改错下面这段
5
#<password_sha256_hex>885551fc11e6ce714c2a947f0fb958945c84804c1cfcff575b67ea795ef357ef</password_sha256_hex>
6

7
cat config/users.xml
8
<clickhouse>
9
    <profiles>
10
        <default>
11
        </default>
12
        <readonly>
13
            <readonly>1</readonly>
14
        </readonly>
15

16
    </profiles>
17
    <users>
18
        <default>
19
            <password>1qaz@WSX</password>
20
            <networks>
21
                <ip>::/0</ip>
22
            </networks>
23
            <profile>default</profile>
24
            <quota>default</quota>
25
            <access_management>1</access_management>
26
            <named_collection_control>1</named_collection_control>
27
        </default>
28
        <nginx>
29
            <password>1qaz@WSX</password>
30
            <networks>
31
                <ip>::/0</ip>
32
            </networks>
33
            <profile>default</profile>
34
            <access_management>1</access_management>
35
            <quota>default</quota>
36
            <access_management>1</access_management>
37
            <named_collection_control>1</named_collection_control>
38
        </nginx>
39
    </users>
40

41
    <quotas>
42
        <default>
43
            <interval>
44
                <duration>3600</duration>
45
                <queries>0</queries>
46
                <errors>0</errors>
47
                <result_rows>0</result_rows>
48
                <read_rows>0</read_rows>
49
                <execution_time>0</execution_time>
50
            </interval>
51
        </default>
52
    </quotas>
53
</clickhouse>

1
docker exec -it clickhouse clickhouse-client --user nginx --password 1qaz@WSX
2
ch_accesslog :)
3
CREATE DATABASE IF NOT EXISTS nginxlogs ENGINE=Atomic;
4

5
CREATE TABLE nginxlogs.nginx_access
6
(
7
    `timestamp` DateTime64(3, 'Asia/Shanghai'),
8
    `server_ip` String,
9
    `domain` String,
10
    `request_method` String,
11
    `status` Int32,
12
    `top_path` String,
13
    `path` String,
14
    `query` String,
15
    `protocol` String,
16
    `referer` String,
17
    `upstreamhost` String,
18
    `responsetime` Float32,
19
    `upstreamtime` Float32,
20
    `duration` Float32,
21
    `request_length` Int32,
22
    `response_length` Int32,
23
    `client_ip` String,
24
    `client_latitude` Float32,
25
    `client_longitude` Float32,
26
    `remote_user` String,
27
    `remote_ip` String,
28
    `xff` String,
29
    `client_city` String,
30
    `client_region` String,
31
    `client_country` String,
32
    `http_user_agent` String,
33
    `client_browser_family` String,
34
    `client_browser_major` String,
35
    `client_os_family` String,
36
    `client_os_major` String,
37
    `client_device_brand` String,
38
    `client_device_model` String,
39
    `createdtime` DateTime64(3, 'Asia/Shanghai')
40
)
41
ENGINE = MergeTree
42
PARTITION BY toYYYYMMDD(timestamp)
43
PRIMARY KEY (timestamp,
44
 server_ip,
45
 status,
46
 top_path,
47
 domain,
48
 upstreamhost,
49
 client_ip,
50
 remote_user,
51
 request_method,
52
 protocol,
53
 responsetime,
54
 upstreamtime,
55
 duration,
56
 request_length,
57
 response_length,
58
 path,
59
 referer,
60
 client_city,
61
 client_region,
62
 client_country,
63
 client_browser_family,
64
 client_browser_major,
65
 client_os_family,
66
 client_os_major,
67
 client_device_brand,
68
 client_device_model
69
)
70
TTL toDateTime(timestamp) + toIntervalDay(30)
71
SETTINGS index_granularity = 8192;

1
# 创建部署目录和docker-compose.yaml
2
mkdir -p /data/server/vector/conf
3
cd /data/server/vector
4
touch access_vector_error.log
5
wget https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb
6
cat <<-EOF > docker-compose.yaml
7
services:
8
  vector:
9
    image: timberio/vector:0.48.X-alpine
10
    container_name: vector
11
    hostname: vector
12
    restart: always
13
    entrypoint: vector --config-dir /etc/vector/conf
14
    ports:
15
      - 8686:8686
16
    volumes:
17
      - /var/log/nginx:/nginx_logs  # 这是需要采集的日志的路径需要挂载到容器内
18
      - ./access_vector_error.log:/tmp/access_vector_error.log
19
      - ./GeoLite2-City.mmdb:/etc/vector/GeoLite2-City.mmdb
20
      - ./conf:/etc/vector/conf
21
      - /etc/localtime:/etc/localtime
22
EOF