Prometheus 安装部署出图（使用grafana）
Posted givenchy_yzl
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Prometheus 安装部署出图（使用grafana）相关的知识，希望对你有一定的参考价值。
一、Prometheus安装与部署
注意：gitlab自带Prometheus，并会启动Prometheus服务，所以会冲突，不要将gitlab与Prometheus部署在一起，否则会出现端口冲突问题
安装服务端
# 下载安装包
[root@promethus ~]# mkdir /prometheus
[root@promethus /opt]# cd /prometheus/
[root@promethus /prometheus]# wget https://github.com/prometheus/prometheus/releases/download/v2.25.0/prometheus-2.25.0.linux-amd64.tar.gz
# 解压
[root@promethus /prometheus]# tar xf prometheus-2.25.0.linux-amd64.tar.gz
[root@promethus /prometheus]# cd prometheus-2.25.0.linux-amd64/
[root@promethus /prometheus/prometheus-2.25.0.linux-amd64]# mv prometheus-2.25.0.linux-amd64/* .
[root@promethus /prometheus]# cd ..
[root@promethus /prometheus]# rm -rf prometheus-2.25.0.linux-amd64*
# 创建用户并授权
[root@promethus /prometheus]# useradd -s /sbin/nologin prometheus -M
[root@promethus /prometheus]# chown -R prometheus.prometheus /prometheus
# 添加环境变量
[root@promethus /prometheus]# vim /etc/profile
export PATH=/prometheus:$PATH
[root@prometheus prometheus]# source /etc/profile
[root@prometheus prometheus]# vim prometheus.yml
.....
- targets: ['192.168.13.17:9090'] #修改为自己的IP
# 配置systemd管理
[root@promethus ~]# vim /usr/lib/systemd/system/prometheus.service
[Unit]
Description=prometheus server daemon
[Service]
ExecStart=/prometheus/prometheus --config.file=/prometheus/prometheus.yml
Restart=on-failure
[Install]
WantedBy=multi-user.target
# 重载启动列表
[root@promethus ~]# systemctl daemon-reload
# 启动
[root@promethus ~]# systemctl start prometheus.service
# 开机自启
[root@promethus ~]# systemctl enable --now prometheus.service
添加监控节点
#其他主机
[root@harbor ~]# cd /opt/
[root@harbor ~]# wget https://github.com/prometheus/node_exporter/releases/download/v1.1.1/node_exporter-1.1.1.linux-amd64.tar.gz
[root@harbor opt]# tar xf node_exporter-1.1.1.linux-amd64.tar.gz -C /usr/local/
[root@harbor opt]# ln -s /usr/local/node_exporter-1.1.1.linux-amd64/ /usr/local/node_exporter
[root@web02 /prometheus_node]# vim /usr/lib/systemd/system/node_exporter.service
[Unit]
Description=prometheus server daemon
[Service]
ExecStart=/usr/local/node_exporter/node_exporter
Restart=on-failure
[Install]
WantedBy=multi-user.target
[root@harbor opt]# systemctl daemon-reload
[root@harbor opt]# systemctl enable --now node_exporter.service
#prometheus服务器
# 定义节点方法一：
static_configs:
- targets: ['localhost:9090','web01:9100','10.0.0.8:9100'] --->定义监控节点
定义节点方法二：
- job_name: 'k8s'
static_configs:
- targets:
- "192.168.13.11:9100"
- "192.168.13.12:9100"
- "192.168.13.13:9100"
- "192.168.13.14:9100"
- "192.168.13.16:9100"
[root@prometheus ~]# systemctl daemon-reload
[root@prometheus ~]# systemctl restart prometheus.service
二、安装后文件说明
目录下文件说明
[root@promethus /prometheus]# ls
console_libraries --->控制台函数库
consoles --->控制台
data --->数据存放目录
LICENSE --->许可证
NOTICE --->通知
prometheus --->启动脚本
prometheus.yml --->主配置文件
promtool --->系统工具
主配置文件说明
[root@promethus /prometheus]# cat prometheus.yml
global: --->全局变量
scrape_interval: 15s # 抓取时间间隔，每隔15秒去抓取一次
evaluation_interval: 15s # 监控数据评估间隔
scrape_configs:
- job_name: 'prometheus' --->定义job名字
static_configs:
- targets: ['localhost:9090','web01:9100','10.0.0.8:9100'] --->定义监控节点
安装后出现的问题
问题：component=“scrape manager” scrape_pool=prometheus target=http://localhost:9090/metrics msg=“Appending scrape report failed” err=“out of bounds”
问题分析：locahost无法解析
问题解决：将localhost 改为自己的主机ip
重启Prometheus
三、Prometheus加grafana（界面优化）
Grafana是一款用Go语言开发的开源数据可视化工具，可以做数据监控和数据统计，带有告警功能。目前使用grafana的公司有很多，如paypal、ebay、intel等。
特点
1、可视化：快速和灵活的客户端图形具有多种选项。面板插件为许多不同的方式可视化指标和日志。
2、报警：可视化地为最重要的指标定义警报规则。Grafana将持续评估它们，并发送通知。
3、通知：警报更改状态时，它会发出通知。接收电子邮件通知。
4、动态仪表盘：使用模板变量创建动态和可重用的仪表板，这些模板变量作为下拉菜单出现在仪表板顶部。
5、混合数据源：在同一个图中混合不同的数据源!可以根据每个查询指定数据源。这甚至适用于自定义数据源。
6、注释：注释来自不同数据源图表。将鼠标悬停在事件上可以显示完整的事件元数据和标记。
7、过滤器：过滤器允许您动态创建新的键/值过滤器，这些过滤器将自动应用于使用该数据源的所有查询。
安装
# 下载
[root@promethus /prometheus]# cd /opt/
[root@promethus /opt]# wget https://mirrors.tuna.tsinghua.edu.cn/grafana/yum/rpm/grafana-7.4.3-1.x86_64.rpm
# 安装
[root@promethus /opt]# yum localinstall grafana-7.4.3-1.x86_64.rpm -y
# 启动并加入开机自启
[root@promethus /opt]# systemctl start grafana-server.service
[root@promethus /opt]# systemctl enable grafana-server.service
# 检查
[root@promethus /opt]# netstat -lntup |grep grafana
tcp6 0 0 :::3000 :::* LISTEN 18889/grafana-serve
# 访问测试
http://192.168.13.17:3000 出现以下界面即为成功
默认管理用户：admin
默认密码：admin
四、grafana连接到Prometheus
五、简单出图测试
方法一：
添加数据源后---->+ -----> import----->upload json file---->将自己写的json文件导入即可
模板见附件
成功后：如下图
方法二：
添加数据源后---->+ -----> import----> 搜索框输入官方GUI码
GUI码来源：grafana.com---->grafana---->dashborad---->数据源选择 Prometheus---->点击模板---->复制任一个图形化界面GUI码即可
成功：如下图
附件：
"annotations":
"list": [
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
]
,
"description": "【中文版本】2020.06.28更新，增加整体资源展示！支持 Grafana6&7，Node Exporter v0.16及以上的版本，优化重要指标展示。包含整体资源展示与资源明细图表：CPU 内存 磁盘 IO 网络等监控指标。https://github.com/starsliao/Prometheus",
"editable": true,
"gnetId": 12884,
"graphTooltip": 0,
"id": 4,
"iteration": 1622769719910,
"links": [
"icon": "external link",
"tags": [],
"targetBlank": true,
"title": "更新node_exporter",
"tooltip": "",
"type": "link",
"url": "https://github.com/prometheus/node_exporter/releases"
,
"icon": "external link",
"tags": [],
"targetBlank": true,
"title": "更新当前仪表板",
"tooltip": "",
"type": "link",
"url": "https://grafana.com/dashboards/8919"
,
"icon": "external link",
"tags": [],
"targetBlank": true,
"title": "StarsL.cn",
"tooltip": "",
"type": "link",
"url": "https://starsl.cn"
,
"asDropdown": true,
"icon": "external link",
"tags": [],
"targetBlank": true,
"title": "",
"type": "dashboards"
],
"panels": [
"collapsed": false,
"datasource": "Prometheus",
"gridPos":
"h": 1,
"w": 24,
"x": 0,
"y": 0
,
"id": 187,
"panels": [],
"title": "资源总览（关联JOB项）当前选中主机：【$show_hostname】实例：$node",
"type": "row"
,
"columns": [],
"datasource": "Prometheus",
"description": "分区使用率、磁盘读取、磁盘写入、下载带宽、上传带宽，如果有多个网卡或者多个分区，是采集的使用率最高的网卡或者分区的数值。",
"fieldConfig":
"defaults":
"custom":
,
"overrides": []
,
"fontSize": "100%",
"gridPos":
"h": 5,
"w": 24,
"x": 0,
"y": 1
,
"id": 185,
"pageSize": 10,
"showHeader": true,
"sort":
"col": 5,
"desc": false
,
"styles": [
"alias": "主机名",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 1,
"link": false,
"linkTooltip": "",
"linkUrl": "",
"mappingType": 1,
"pattern": "nodename",
"thresholds": [],
"type": "string",
"unit": "bytes"
,
"alias": "IP（链接到明细）",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": false,
"linkTooltip": "浏览主机明细",
"linkUrl": "/d/9CWBz0bik/node-exporter?orgId=1&var-job=$job&var-hostname=All&var-node=$__cell&var-device=All",
"mappingType": 1,
"pattern": "instance",
"thresholds": [],
"type": "number",
"unit": "short"
,
"alias": "内存",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"mappingType": 1,
"pattern": "Value #B",
"thresholds": [],
"type": "number",
"unit": "bytes"
,
"alias": "CPU核",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"mappingType": 1,
"pattern": "Value #C",
"thresholds": [],
"type": "number",
"unit": "short"
,
"alias": " 运行时间",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #D",
"thresholds": [],
"type": "number",
"unit": "s"
,
"alias": "分区使用率*",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #E",
"thresholds": [
"70",
"85"
],
"type": "number",
"unit": "percent"
,
"alias": "CPU使用率",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #F",
"thresholds": [
"70",
"85"
],
"type": "number",
"unit": "percent"
,
"alias": "内存使用率",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #G",
"thresholds": [
"70",
"85"
],
"type": "number",
"unit": "percent"
,
"alias": "磁盘读取*",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #H",
"thresholds": [
"10485760",
"20485760"
],
"type": "number",
"unit": "Bps"
,
"alias": "磁盘写入*",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #I",
"thresholds": [
"10485760",
"20485760"
],
"type": "number",
"unit": "Bps"
,
"alias": "下载带宽*",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #J",
"thresholds": [
"30485760",
"104857600"
],
"type": "number",
"unit": "bps"
,
"alias": "上传带宽*",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #K",
"thresholds": [
"30485760",
"104857600"
],
"type": "number",
"unit": "bps"
,
"alias": "5m负载",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #L",
"thresholds": [],
"type": "number",
"unit": "short"
,
"alias": "",
"align": "right",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "hidden",
"unit": "short"
],
"targets": [
"expr": "node_uname_infojob=~\\"$job\\" - 0",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "主机名",
"refId": "A"
,
"expr": "sum(time() - node_boot_time_secondsjob=~\\"$job\\")by(instance)",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "运行时间",
"refId": "D"
,
"expr": "node_memory_MemTotal_bytesjob=~以上是关于Prometheus 安装部署出图（使用grafana）的主要内容，如果未能解决你的问题，请参考以下文章
Prometheus+Grafana系统部署，linuxflink的监控与告警