一、基本结构
- node_exporter：宿主机监控插件；
- cadvisor：容器监控插件；
- prometheus：支持外部数据库也可使用内部数据库；
- grafana：集成报警功能；可替代altermanager功能；
二、安装部署
step 1 节点启动node_exporter和cadvisor镜像
|
监控插件镜像文件 文件提取码：m5jg |
|
docker run -d -p 宿主机端口:9100 -v "/:/host:ro,rslave" --name=容器名 镜像名称 --path.rootfs /host #启动node_exporter docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:rw --volume=/sys:/sys:ro --volume=/var/lib/docker/:/var/lib/docker:ro --publish=宿主机端口:8080 --detach=true --name=cadvisor google/cadvisor:latest #启动cadvisor |
[root@docker01 ~]# docker pull quay.io/prometheus/node-exporter
Using default tag: latest
......
Status: Downloaded newer image for quay.io/prometheus/node-exporter:latest
quay.io/prometheus/node-exporter:latest
[root@docker01 ~]# docker pull google/cadvisor:latest
latest: Pulling from google/cadvisor
......
Status: Image is up to date for google/cadvisor:latest
docker.io/google/cadvisor:latest
[root@docker01 ~]# docker run -d -p 9100:9100 -v "/:/host:ro,rslave" --name=node_exporter quay.io/prometheus/node-exporter --path.rootfs /host
c5d94dc163b491e1bede806136bcde7bb4ce20f1b48b4b6709e6e3bd5c4afddd
[root@docker01 ~]# docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:rw --volume=/sys:/sys:ro --volume=/var/lib/docker/:/var/lib/docker:ro --publish=8080:8080 --detach=true --name=cadvisor google/cadvisor:latest
18303c955b3314912e6355f3410de4b6123431f7259458ab74593407f1ab84c8
[root@docker01 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
18303c955b33 google/cadvisor:latest "/usr/bin/cadvisor -…" 2 minutes ago Up 2 minutes 0.0.0.0:8080->8080/tcp cadvisor
c5d94dc163b4 quay.io/prometheus/node-exporter "/bin/node_exporter …" 8 minutes ago Up 7 minutes 0.0.0.0:9100->9100/tcp node_exporter[root@docker02 ~]# docker pull quay.io/prometheus/node-exporter
Using default tag: latest
......
Status: Downloaded newer image for quay.io/prometheus/node-exporter:latest
quay.io/prometheus/node-exporter:latest
[root@docker02 ~]# docker pull google/cadvisor:latest
latest: Pulling from google/cadvisor
......
Status: Downloaded newer image for google/cadvisor:latest
docker.io/google/cadvisor:latest
[root@docker02 ~]# docker run -d -p 9100:9100 -v "/:/host:ro,rslave" --name=node_exporter quay.io/prometheus/node-exporter --path.rootfs /host
ab842e32386c1ab29fa0d22188f3292ff1444a01aeff22c4f7e62055e6090452
[root@docker02 ~]# docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:rw --volume=/sys:/sys:ro --volume=/var/lib/docker/:/var/lib/docker:ro --publish=8080:8080 --detach=true --name=cadvisor google/cadvisor:latest
d80a3824b17898d7bb05c7c15652b18a255d03600108db829ca92b7576a6dcc9
[root@docker02 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
d80a3824b178 google/cadvisor:latest "/usr/bin/cadvisor -…" About a minute ago Up About a minute 0.0.0.0:8080->8080/tcp cadvisor
ab842e32386c quay.io/prometheus/node-exporter "/bin/node_exporter …" 7 minutes ago Up 7 minutes 0.0.0.0:9100->9100/tcp node_exporter
step 2 下载prometheus镜像
|
tar xf 压缩包 cd 目录 |
[root@consul ~]# ll prometheus-2.19.1.linux-amd64.tar.gz
-rw-r--r-- 1 root root 64156261 Jun 23 11:52 prometheus-2.19.1.linux-amd64.tar.gz
[root@consul ~]# tar xf prometheus-2.19.1.linux-amd64.tar.gz
[root@consul ~]# ls
anaconda-ks.cfg prometheus-2.19.1.linux-amd64.tar.gz
prometheus-2.19.1.linux-amd64
[root@consul ~]# mkdir /application
[root@consul ~]# mv prometheus-2.19.1.linux-amd64 /application/
[root@consul ~]# cd /application/
[root@consul /application]# ln -s prometheus-2.19.1.linux-amd64/ prometheus
[root@consul /application]# ls
prometheus prometheus-2.19.1.linux-amd64
[root@consul /application]# cd prometheus
[root@consul /application/prometheus]# ls
console_libraries LICENSE prometheus promtool
consoles NOTICE prometheus.yml tsdbstep 3 配置prometheus
prometheus默认仅监控自己，默认端口9090。
|
#prometheus.yml文件 global:
scrape_interval: 15s #数据采样间隔时间
evaluation_interval: 15s #数据采样间隔时间
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'cadvisor' #设置监控任务名
static_configs:
- targets: ['IP地址1:端口','IP地址2:端口',...]
- job_name: 'node_exporter'
static_configs:
- targets: ['IP地址1:端口','IP地址2:端口',...]
|
[root@consul /application/prometheus]# grep -Ev '^$|^[ ]+#|^#' prometheus.yml
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
alerting:
alertmanagers:
- static_configs:
- targets:
rule_files:
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['localhost:9090']
- job_name: 'cadvisor'
static_configs:
- targets: ['10.0.0.110:8080','10.0.0.120:8080']
- job_name: 'node_exporter'
static_configs:
- targets: ['10.0.0.110:9100','10.0.0.120:9100']step 4 安装prometheus
| ./prometheus --config.file="/目录/prometheus.yml" |
[root@consul /application/prometheus]# ./prometheus --config.file="./prometheus.yml"
level=info ts=2020-06-23T05:18:57.221Z caller=main.go:302 msg="No time or size retention was set so using the default time retention" duration=15d
......
step 5 安装并启动grafana
grafana默认管理员的账号和密码为：admin
|
yum localinstall -y 安装包 #安装grafana systemctl start grafana-server #启动grafana |
[root@consul /application]# yum localinstall -y grafana-7.0.3-1.x86_64.rpm
Loaded plugins: fastestmirror
Examining grafana-7.0.3-1.x86_64.rpm: grafana-7.0.3-1.x86_64
Marking grafana-7.0.3-1.x86_64.rpm to be installed
......
xorg-x11-font-utils.x86_64 1:7.5-21.el7
xorg-x11-server-utils.x86_64 0:7.7-20.el7
Complete!
[root@consul /application]# systemctl start grafana-server.service
[root@consul /application]# netstat -lntp | grep grafana
tcp6 0 0 :::3000 :::* LISTEN 8533/grafana-server 
step 6 grafana连接prometheus
| Configuration -> Data Source -> Add data source -> Prometheus -> Select |
step 7 grafana导入dashboard
获取模版
导入模版
| Create -> Import -> Upload .json file -> Prometheus -> Import |
测试
|
示例模版 文件提取码：iy5t |
| ab -n 总访问次数 -c 单次请求数 http://域名:端口/资源 |
[root@docker01 ~]# docker run -d -P kod:v7.1 123456
b581922d5acb0216f841c77680416647e6249d522f3fc5201fde7fd9ef882bcd
[root@docker01 ~]# docker ps -l
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b581922d5acb kod:v7.1 "/bin/bash /init.sh …" 8 seconds ago Up 5 seconds 0.0.0.0:32769->22/tcp, 0.0.0.0:32768->80/tcp beautiful_sammet[root@docker01 ~]# yum install -y httpd-tools.x86_64
......
Installed:
httpd-tools.x86_64 0:2.4.6-93.el7.centos
Complete!
[root@docker01 ~]# ab -n 1000000 -c 500 http://10.0.0.110:32768/index.php
This is ApacheBench, Version 2.3 <$Revision: 1430300 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 10.0.0.110 (be patient)
^C
Server Software: nginx/1.18.0
Server Hostname: 10.0.0.110
Server Port: 32768
Document Path: /index.php
Document Length: 0 bytes
Concurrency Level: 500
Time taken for tests: 862.611 seconds
Complete requests: 77958
Failed requests: 3135
(Connect: 0, Receive: 0, Length: 3135, Exceptions: 0)
Write errors: 0
Non-2xx responses: 77958
Total transferred: 69814695 bytes
HTML transferred: 498075 bytes
Requests per second: 90.37 [#/sec] (mean)
Time per request: 5532.538 [ms] (mean)
Time per request: 11.065 [ms] (mean, across all concurrent requests)
Transfer rate: 79.04 [Kbytes/sec] received
Connection Times (ms)
min mean[+/-sd] median max
Connect: 0 6 97.2 0 3011
Processing: 34 5452 7865.8 2797 91267
Waiting: 34 5452 7865.8 2796 91267
Total: 45 5459 7874.0 2797 91269
Percentage of the requests served within a certain time (ms)
50% 2797
66% 4362
75% 4935
80% 6173
90% 10765
95% 17094
98% 32900
99% 44971
100% 91269 (longest request)