Install Prometheus + Grafana

2018-6-29

以下方式可以採用LXC方式拆分 Prometheus /Grafana/以及各node_exproter

新增使用者:

useradd --no-create-home --shell /bin/false prometheus
useradd --no-create-home --shell /bin/false node_exporter

建立資料夾:

mkdir /etc/prometheus
mkdir /var/lib/prometheus

給予群組:

chown prometheus:prometheus /etc/prometheus
chown prometheus:prometheus /var/lib/prometheus

下載Prometheus:

wget https://github.com/prometheus/prometheus/releases/download/v2.3.1/prometheus-2.3.1.linux-amd64.tar.gz

最新版:https://prometheus.io/download/

解壓縮之後複製:

tar zxvf prometheus-XXX.tar.gz

cp prometheus-XXX/prometheus /usr/local/bin/
cp prometheus-XXX/promtool /usr/local/bin/
cp -r prometheus-XXX/consoles /etc/prometheus
cp -r prometheus-XXX/console_libraries /etc/prometheus

給予群組:

chown prometheus:prometheus /usr/local/bin/prometheus
chown prometheus:prometheus /usr/local/bin/promtool
chown -R prometheus:prometheus /etc/prometheus/consoles
chown -R prometheus:prometheus /etc/prometheus/console_libraries

建立設定檔:

nano /etc/prometheus/prometheus.yml
global:
  scrape_interval:     20s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.

alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093
remote_write:
  - url: "http://10.0.1.167:8086/api/v1/prom/write?db=Prometheus"

remote_read:
  - url: "http://10.0.1.167:8086/api/v1/prom/read?db=Prometheus"
rule_files:
scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  - job_name: 'node'
    scrape_interval: 10s
    scrape_timeout: 10s
    static_configs:
      - targets: ['10.0.1.60:9100']
      - targets: ['10.0.1.93:9100']
      - targets: ['10.0.1.94:9100']
      - targets: ['10.0.1.95:9100']
      - targets: ['10.0.1.104:9100']
      - targets: ['10.0.1.105:9100']
      - targets: ['10.0.1.106:9100']
      - targets: ['10.0.1.108:9100']
#      - targets: ['10.0.1.110:9100']
      - targets: ['10.0.1.113:9100']
      - targets: ['10.0.1.114:9100']
      - targets: ['10.0.1.118:9100']
      - targets: ['10.0.1.119:9100']
      - targets: ['XX.XX.XX.XX:9100']
      - targets: ['XX.XX.XX.XX:9100']
      - targets: ['XX.XX.XX.XX:9100']
      - targets: ['XX.XX.XX.XX:9100']
#  - job_name: 'Proxmox'
#    metrics_path: /pve
#    static_configs:
#      - targets: ['10.0.1.110:9221']
#      - targets: ['10.0.1.111:9221']
#      - targets: ['10.0.1.112:9221']
#      - targets: ['10.0.1.113:9221']
#      - targets: ['10.0.1.114:9221']
#      - targets: ['10.0.1.115:9221']
  - job_name: 'netdata'
    metrics_path: /api/v1/allmetrics
    params:
      format: [ prometheus ]
    static_configs:
      - targets: ['10.0.1.60:19999']
      - targets: ['10.0.1.93:19999']
      - targets: ['10.0.1.94:19999']
      - targets: ['10.0.1.95:19999']
      - targets: ['10.0.1.104:19999']
      - targets: ['10.0.1.105:19999']
      - targets: ['10.0.1.106:19999']
      - targets: ['10.0.1.108:19999']
#      - targets: ['10.0.1.110:19999']
      - targets: ['10.0.1.113:19999']
      - targets: ['10.0.1.114:19999']
      - targets: ['10.0.1.118:19999']
      - targets: ['10.0.1.119:19999']
  - job_name: 'Windows Status'
    static_configs:
      - targets: ['10.0.5.7:9182']
      - targets: ['10.0.5.18:9182']
      - targets: ['10.0.5.61:9182']
      - targets: ['10.0.5.64:9182']
  - job_name: 'Minecraft'
    static_configs:
      - targets: ['10.0.1.186:9225']
  - job_name: 'Nginx'
    static_configs:
      - targets: ['10.0.1.60:9145']
  - job_name: 'smokeping_prober'
    static_configs:
      - targets: ['10.0.1.169:9374']
  - job_name: 'cadvisor'
    static_configs:
      - targets: ['10.0.1.108:8001']
  - job_name: 'Container-Exporter'
    static_configs:
      - targets: ['10.0.1.108:9104']
  - job_name: 'dump1090exporter'
    static_configs:
      - targets: ['10.0.1.108:9105']
  - job_name: 'Web_DB'
    static_configs:
      - targets: ['10.0.1.60:9104']
        labels:
          alias: Web_DB
  - job_name: 'MTProto_Service'
    static_configs:
      - targets: ['10.0.1.179:3129']
        labels:
          alias: TW_MTP
      - targets: ['XX.XX.XX.XX:3129']
        labels:
          alias: Toyko_MTP
      - targets: ['XX.XX.XX.XX:3129']
        labels:
          alias: Osaka_MTP
  - job_name: 'fortigate_exporter'
    metrics_path: /probe
    static_configs:
      - targets:
        - https://114.domain
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
        # Drop the https:// and port (if specified) for the 'instance=' label
        regex: '(?:.+)(?::\/\/)([^:]*).*'
      - target_label: __address__
        replacement: '10.0.1.174:9710'
  - job_name: 'blackbox'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
     - targets: ['1.1.1.1']
       labels:
         alias: 1.1.1.1
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.0.100.2:9115

建立服務:

nano /etc/systemd/system/prometheus.service

[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
    --config.file /etc/prometheus/prometheus.yml \
    --storage.tsdb.path /var/lib/prometheus/ \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries
選配:--web.listen-address=:9091 \
    --storage.tsdb.retention.time=60d \
    --storage.tsdb.retention=60d \

[Install]
WantedBy=multi-user.target

重新載入並開啟服務:

systemctl daemon-reload
systemctl start prometheus
systemctl status prometheus
systemctl enable prometheus

下載node_exproter:

wget https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz
#最新:https://prometheus.io/download/
#or
apt install prometheus-node-exporter 即可快速安裝

解壓縮之後複製到指定位置:

tar zxvf node_exporter-0.17.0-rc.0.linux-amd64.tar.gz
mkdir /opt/node_exporter/
cp node_exporter-0.17.0-rc.0.linux-amd64/node_exporter /opt/node_exporter/
chown -R node_exporter:node_exporter /opt/node_exporter/

建立服務:

nano /etc/systemd/system/node_exporter.service

[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/opt/node_exporter/node_exporter

[Install]
WantedBy=multi-user.target

重新載入並開啟服務:

systemctl daemon-reload
systemctl start node_exporter
systemctl status node_exporter
systemctl enable node_exporter

Node_exporter 額外分支 text_collector(須預先安裝ipmitool):

apt-get install python-sponge
crontab -e

* * * * * /opt/node_exporter/text_collector/nvme_temp.bsh |/opt/node_exporter/text_collector/nvme_temp.prom
* * * * * /opt/node_exporter/text_collector/mellanox_hca_temp.bsh | /opt/node_exporter/text_collector/mellanox_hca_temp.prom
* * * * * /usr/bin/ipmitool sensor | /opt/node_exporter/text_collector/ipmitool.bsh > /opt/node_exporter/text_collector/ipmitool.prom

nano /etc/default/prometheus-node-exporter
ARGS="--collector.textfile.directory=/opt/node_exporter/text_collector"

bsh/prom File: Download

建立 pve exporter:

apt install virtualenv
virtualenv /opt/prometheus-pve-exporter
/opt/prometheus-pve-exporter/bin/pip install prometheus-pve-exporter

mkdir -p /etc/prometheus
nano /etc/prometheus/pve.yml
default:
    user: prometheus@pve
    password: admin
    verify_ssl: false

cat < /etc/systemd/system/prometheus-pve-exporter.service
[Unit]
Description=Prometheus exporter for Proxmox VE
Documentation=https://github.com/znerol/prometheus-pve-exporter
[Service]
Restart=always
User=root
ExecStart=/opt/prometheus-pve-exporter/bin/pve_exporter /etc/prometheus/pve.yml
[Install]
WantedBy=multi-user.target
EOF

新增監控使用者與服務:

pveum useradd prometheus@pve -comment "prometheus plugin user"
pveum passwd prometheus@pve
pveum groupadd monitors
pveum aclmod / -group monitors -role PVEAuditor
pveum usermod prometheus@pve -group monitors
pveum aclmod / -user prometheus@pve -role PVEAuditor

systemctl daemon-reload
systemctl start prometheus-pve-exporter
apt install prometheus-node-exporter
systemctl enable prometheus-node-exporter
systemctl start prometheus-node-exporter

snmp_exporter: 安裝:

virtualenv /opt/prometheus-snmp-exporter
/opt/prometheus-snmp-exporter/bin/pip install snmp_exporter

指定ver.:

virtualenv --python=python3 /opt/prometheus-pve-exporter
virtualenv --python=python /opt/prometheus-snmp-exporter

建立服務:

cat <<EOF> /etc/systemd/system/prometheus-snmp-exporter.service

[Unit]

Description=Prometheus exporter for SNMP
Documentation=https://github.com/prometheus/snmp_exporter
[Service]

Restart=always
User=root

ExecStart=/opt/prometheus-snmp-exporter/bin/snmp_exporter 
[Install]
WantedBy=multi-user.target

EOF

重新載入並開啟服務:

systemctl daemon-reload
systemctl start prometheus-snmp-exporter
systemctl status prometheus-snmp-exporter
systemctl enable prometheus-snmp-exporter

Prometheus metric library for Nginx:

下載來源:

git clone http://www.github.com/knyar/nginx-lua-prometheus

到 Web Server 內的 Nginx 新增:

nano /etc/nginx/nginx.conf
區段http{ 加入

lua_shared_dict prometheus_metrics 10M;
lua_package_path "/path/to/nginx-lua-prometheus/?.lua";
init_by_lua '
  prometheus = require("prometheus").init("prometheus_metrics")
  metric_requests = prometheus:counter(
    "nginx_http_requests_total", "Number of HTTP requests", {"host", "status"})
  metric_latency = prometheus:histogram(
    "nginx_http_request_duration_seconds", "HTTP request latency", {"host"})
  metric_connections = prometheus:gauge(
    "nginx_http_connections", "Number of HTTP connections", {"state"})
';
log_by_lua '
  metric_requests:inc(1, {ngx.var.server_name, ngx.var.status})
  metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name})
';

新增 Site:

server {
  listen 9145;
  allow 192.168.0.0/16;
  deny all;
  location /metrics {
    content_by_lua '
      metric_connections:set(ngx.var.connections_reading, {"reading"})
      metric_connections:set(ngx.var.connections_waiting, {"waiting"})
      metric_connections:set(ngx.var.connections_writing, {"writing"})
      prometheus:collect()
    ';
  }
}

WMI_exporter: Download

msiexec /i c:\users\user\downloads\wmi_exporter-0.5.0-amd64.msi ENABLED_COLLECTORS=cpu,cs,logical_disk,net,os,system,textfile TEXTFILE_DIR="C:\wmi_exporter"

安裝完成後:

enable remote powershell:  Enable-PSRemoting -Force
add optional feature openssh server

如何塞 smartctl 裡面的 disk temperature 到 prometheus 裡面:
1. 首先要有 prometheus server 建立好
2. target node 必須要安裝一下的 package

apt install lsscsi hddtemp prometheus-node-exporter

3.如果要完整的 smartmon info 可以啟用 prometheus-node-exporter-smartmon.service 然後修改

ExecStart = ExecStart=/bin/bash -c "/usr/share/prometheus-node-exporter/smartmon.sh | /usr/sbin/sponge /opt/node_exporter/text_collector/smartmon.prom"

4.設定你要收集的 args... 我的如下 (我不要收集太多數據所以我都是刪去法這個是我的 ceph node 沒有 zfs pool)

nano /etc/default/prometheus-node-exporter

ARGS="--collector.textfile.directory=/opt/node_exporter/text_collector --no-collector.arp --no-collector.infiniband --no-collector.mdadm --no-collector.nfs --no-collector.nfsd --no-collector.xfs --no-collector.zfs"

5.新增 disktemp_metrics

nano /opt/node_exporter/disktemp_metrics.sh
#!/bin/bash

echo "# HELP disk_temperature SMART metric celcius"
echo "# TYPE disk_temperature gauge"

array=$(lsscsi|awk '{print $6}' )

/usr/sbin/hddtemp $array | awk -F ":|/|°" '{print "disk_temperature{device=\""$3"\"}" $5}'
#/usr/sbin/hddtemp /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg | awk -F ":|/|°" '{print "disk_temperature{device=\""$3"\"}" $5}'

6.新增排程

crontab -e * * * * * bash -c "/opt/node_exporter/disktemp_metrics.sh | /usr/bin/sponge /opt/node_exporter/text_collector/disktemp_metrics.prom"

安裝 smokeping_prober exporter:

wget https://github.com/SuperQ/smokeping_prober/releases/download/v0.3.0/smokeping_prober-0.3.0.linux-amd64.tar.gz
tar zxvf smokeping_prober-0.3.0.linux-amd64.tar.gz
mkdir /opt/smokeping_prober/
mv smokeping_prober-0.3.0.linux-amd64/smokeping_prober /opt/smokeping_prober/

新增服務:

nano /etc/systemd/system/smokeping_prober.service

[Unit]
Description=smokeping_prober
Wants=network-online.target
After=network-online.target

[Service]
User=root
Group=root
Type=simple
Restart=always
ExecStart=/opt/smokeping_prober/smokeping_prober --ping.interval=30s 127.0.0.1

[Install]
WantedBy=multi-user.target

啟用服務:

systemctl daemon-reload
systemctl start smokeping_prober
systemctl status smokeping_prober
systemctl enable smokeping_prober

raspberrypi_exporter:
安裝服務:

apt install prometheus-node-exporter
curl -fsSL "https://raw.githubusercontent.com/fahlke/raspberrypi_exporter/master/installer.sh" | sudo bash

編輯服務:

nano /lib/systemd/system/prometheus-node-exporter.service

#在此行添加內容如下
ExecStart=/usr/bin/prometheus-node-exporter $ARGS --collector.textfile.directory=/var/lib/node_exporter/textfile_collector

重啟服務:

systemctl daemon-reload
service prometheus-node-exporter restart
systemctl status prometheus-node-exporter

dump1090-exporter:

docker run -it --rm clawsicus/dump1090exporter
#or
docker pull clawsicus/dump1090exporter

docker run -p 9105:9105   --detach   --restart always clawsicus/dump1090exporter --url=http://dump1090_IP:8080  --latitude=緯度  --longitude=經度

fortigate_exporter:

wget https://github.com/bluecmd/fortigate_exporter/releases/download/v1.5.0/fortigate-exporter.linux.amd64

tar zxvf fortigate-exporter.linux.amd64

新建設定檔:

nano fortigate-key.yaml

"https://my-fortigate":
  token: api-key-goes-here

啟動服務:

./fortigate-exporter.linux.amd64 -auth-file ~/fortigate-key.yaml -insecure true

建立域名憑證:

openssl genrsa -des3 -out RootCA.key 2048
openssl req -new -key RootCA.key -out RootCA.req
openssl x509 -extfile extfile.cnf -req -days 3650 -sha1 -signkey RootCA.key -in RootCA.req -out RootCA.crt

建立extfile.cnf:

subjectAltName = @alt_names
[ alt_names ]
DNS.1 = domain.com
DNS.2 = domain2.com
DNS.3 = domain3.com

檔案匯入Forti後將網頁證書更換為產生出的證書

產生 User API Key:

FortiGate REST API Token Authentication

blackbox_exporter:

wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.19.0/blackbox_exporter-0.19.0.linux-amd64.tar.gz

tar zxvf blackbox_exporter-0.19.0.linux-amd64.tar.gz

mkdir /opt/blackbox_exporter
mv blackbox_exporter  /opt/blackbox_exporter/
mv blackbox.yml /opt/blackbox_exporter/

建立服務:

nano /etc/systemd/system/blackbox_exporter.service

[Unit]
Description=blackbox_exporter
Wants=network-online.target
After=network-online.target

[Service]
User=root
Type=simple
Restart=always
ExecStart= /opt/blackbox_exporter/blackbox_exporter --config.file /opt/blackbox_exporter/blackbox.yml

[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl restart blackbox_exporter
systemctl status blackbox_exporter

安裝 Grafana:
安裝:

wget https://dl.grafana.com/oss/release/grafana_6.5.2_amd64.deb
sudo dpkg -i grafana_6.5.2_amd64.deb

啟用服務:

service grafana-server start
update-rc.d grafana-server defaults


or

systemctl daemon-reload
systemctl start grafana-server
systemctl status grafana-server
systemctl enable grafana-server.service

安裝CloudFlare 插件:

grafana-cli plugins install cloudflare-app

篩選:

node_disk_io_now{device!~"md.*|dm.*|loop.*|rbd.*|zd.*"}

Node Exporter 客製面板:Node Exporter-1584207125162.json

MTproto:Telegram MTProto Proxy (mtg) Status-1584207043944.json

Docker 使用方式: Docker:prometheus and exporter pull

阅读剩余

THE END