25、Prometheus监控系统

# 部署kube-state-metrics，用于统计Kubernetes集群之上各种各样的计数器类的数据
[root@node1 prometheus]# kubectl apply -f  kube-state-metrics/
deployment.apps/kube-state-metrics created
configmap/kube-state-metrics-config created
serviceaccount/kube-state-metrics created
clusterrole.rbac.authorization.k8s.io/kube-state-metrics created
role.rbac.authorization.k8s.io/kube-state-metrics-resizer created
clusterrolebinding.rbac.authorization.k8s.io/kube-state-metrics created
rolebinding.rbac.authorization.k8s.io/kube-state-metrics created
service/kube-state-metrics created
# 查看kube-state-metrics的service
[root@node1 ~]# kubectl get svc -n kube-system  kube-state-metrics 
NAME                 TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
kube-state-metrics   ClusterIP   10.107.118.157   <none>        8080/TCP,8081/TCP   17m
# 测试kube-state-metrics是否部署成功，一旦有metrics返回，那就说明我们的kube-state-metrics部署成功，它会以KV的形式展示
[root@node1 ~]# curl -I -m 10 -o /dev/null -s -w %{http_code} 10.107.118.157:8080/metrics
200

# 部署node-exporter，用于统计节点的相关指标数据，部署之前需要先将node-exporter-ds.yml添加master节点的污点，否则master节点将不会调度
[root@node1 prometheus]# kubectl apply -f node-exporter/
daemonset.apps/node-exporter created
service/node-exporter created
# 测试node_exporter是否部署成功，一旦有metrics返回，那就说明我们的node_exporter部署成功，它会以KV的形式展示
[root@node1 ~]# curl -I -m 10 -o /dev/null -s -w %{http_code} localhost:9100/metrics
200

# 安装AlterManager，AlterManager需要用到持久存储，所以先创建pv
[root@node1 ~]# yum install -y nfs-utils
[root@node2 ~]# yum install -y nfs-utils
[root@node3 ~]# yum install -y nfs-utils
[root@node1 ~]# mkdir -p /data/volumes{1..3}
[root@node1 ~]# cat /etc/exports
/data/volumes1 172.16.0.0/16(rw,sync,no_root_squash)
/data/volumes2 172.16.0.0/16(rw,sync,no_root_squash)
[root@node1 ~]# exportfs -rav   
exporting 172.16.0.0/16:/data/volumes1
exporting 172.16.0.0/16:/data/volumes2
[root@node1 ~]# systemctl start nfs
# 创建pv
[root@node1 ~]# cat altermanager-pv.yaml 
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-nfs-v1
spec:
  capacity:
    storage: 5Gi
  persistentVolumeReclaimPolicy: Delete # pvc删除策略
  accessModes:
  - ReadWriteOnce
  - ReadOnlyMany
  - ReadWriteMany
  nfs:
    path: /data/volumes1
    server: 172.16.1.2
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pv-nfs-v2
spec:
  capacity:
    storage: 5Gi
  persistentVolumeReclaimPolicy: Delete # pvc删除策略
  accessModes:
  - ReadWriteOnce
  - ReadOnlyMany
  - ReadWriteMany
  nfs:
    path: /data/volumes2
    server: 172.16.1.2
# AlterManager默认使用的storageClass，所以我们需要将其注释掉
[root@node1 alertmanager]# cat alertmanager-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: alertmanager
  namespace: kube-system
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: EnsureExists
spec:
# storageClassName: standard # 注释掉存储类
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: "2Gi"
# 为了能让AlterManager在集群外部能访问，修改AlterManager的service类型为NodePort
[root@node1 prometheus]# cat alertmanager/alertmanager-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: alertmanager
  namespace: kube-system
  labels:
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
    kubernetes.io/name: "Alertmanager"
spec:
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: 9093
      nodePort: 30080
  selector:
    k8s-app: alertmanager
  type: "NodePort"
# 创建AlterManager
[root@node1 prometheus]# kubectl apply -f alertmanager/
configmap/alertmanager-config created
deployment.apps/alertmanager created
persistentvolumeclaim/alertmanager created
service/alertmanager created
[root@node1 ~]# curl -I -m 10 -o /dev/null -s -w %{http_code} 172.16.1.2:30080
200

# 部署prometheus，修改prometheus的Service的service为NodePort，让其在集群外可以访问
[root@node1 prometheus]# cat prometheus-service.yaml
kind: Service
apiVersion: v1
metadata:
  name: prometheus
  namespace: kube-system
  labels:
    kubernetes.io/name: "Prometheus"
    kubernetes.io/cluster-service: "true"
    addonmanager.kubernetes.io/mode: Reconcile
spec:
  ports:
    - name: http
      port: 9090
      protocol: TCP
      targetPort: 9090
      nodePort: 30090
  selector:
    k8s-app: prometheus
  type: "NodePort"
# 此处没有存储类，所以注释掉prometheus的存储类额
[root@node1 prometheus]# tail -6 prometheus-statefulset.yaml 
#      storageClassName: standard
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: "3Gi"
# 部署prometheus
[root@node1 prometheus]# kubectl apply -f prometheus/
configmap/prometheus-config created
serviceaccount/prometheus created
clusterrole.rbac.authorization.k8s.io/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created
service/prometheus created
statefulset.apps/prometheus created

# 生成一个密钥
[root@node1 k8s-prometheus-adapter]# openssl genrsa -out serving.key 2048
# 生产一个证书签署请求
[root@node1 k8s-prometheus-adapter]# openssl req -new -key serving.key -out serving.csr -subj "/CN=serving"
# 使用kubernetes自己的CA签署证书
[root@node1 k8s-prometheus-adapter]# openssl x509 -req -in serving.csr -CA /etc/kubernetes/pki/ca.crt -CAkey /etc/kubernetes/pki/ca.key -CAcreateserial -out serving.crt -days 36500
# 创建名称空间
[root@node1 k8s-prometheus-adapter]# kubectl create ns custom-metrics
# 基于创建出的证书做一个secret
[root@node1 ~]# kubectl create secret generic cm-adapter-serving-certs --from-file=/etc/kubernetes/k8s-prometheus-adapter/serving.key --from-file=/etc/kubernetes/k8s-prometheus-adapter/serving.crt -n custom-metrics
[root@node1 ~]# kubectl get secrets -n custom-metrics cm-adapter-serving-certs -o yaml
# 修改custom-metrics-apiserver-deployment.yaml的配置
[root@node1 manifests]# grep 'prometheus-url' custom-metrics-apiserver-deployment.yaml  
        - --prometheus-url=http://prometheus.kube-system.svc:9090/  # 默认是在prom名称空间，这里需要修改，我们的prometheus在kube-system
# 部署k8s-prometheus-adapter
[root@node1 ~]# kubectl apply -f k8s-prometheus-adapter/manifests/
clusterrolebinding.rbac.authorization.k8s.io/custom-metrics:system:auth-delegator created
rolebinding.rbac.authorization.k8s.io/custom-metrics-auth-reader created
deployment.apps/custom-metrics-apiserver created
clusterrolebinding.rbac.authorization.k8s.io/custom-metrics-resource-reader created
serviceaccount/custom-metrics-apiserver created
service/custom-metrics-apiserver created
apiservice.apiregistration.k8s.io/v1beta1.custom.metrics.k8s.io created
clusterrole.rbac.authorization.k8s.io/custom-metrics-server-resources created
configmap/adapter-config created
clusterrole.rbac.authorization.k8s.io/custom-metrics-resource-reader created
clusterrolebinding.rbac.authorization.k8s.io/hpa-controller-custom-metrics created
# 查看自定义Api群组是否成功
[root@node1 ~]# kubectl api-versions |grep custom.metrics
custom.metrics.k8s.io/v1beta1
# 查看各组件是否运行正常
[root@node1 ~]# kubectl get all -n custom-metrics 
NAME                                            READY   STATUS    RESTARTS   AGE
pod/custom-metrics-apiserver-7666fc78cc-56mq8   1/1     Running   0          103s

NAME                               TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE
service/custom-metrics-apiserver   ClusterIP   10.98.164.69   <none>        443/TCP   104s

NAME                                       READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/custom-metrics-apiserver   1/1     1            1           104s

NAME                                                  DESIRED   CURRENT   READY   AGE
replicaset.apps/custom-metrics-apiserver-7666fc78cc   1         1         1       104s
# 测试自定义资源指标，借助于kubectl使用原始格式直接获取Api群组下面的所有resources
[root@node1 ~]# yum install -y jq  # 安装jq对json格式的数据进行解析
[root@node1 ~]# kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq .|less
{
  "kind": "APIResourceList",
  "apiVersion": "v1",
  "groupVersion": "custom.metrics.k8s.io/v1beta1",
  "resources": [
    {
      "name": "jobs.batch/kube_endpoint_created",
      "singularName": "",
      "namespaced": true,
      "kind": "MetricValueList",
      "verbs": [
        "get"
      ]
    },
...
# 获取指定的资源字段
[root@node1 ~]# kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1" | jq .resources[].name
"namespaces/kube_pod_container_status_waiting_reason"
"namespaces/kube_pod_start_time"
"jobs.batch/kubelet_container_log_filesystem_used_bytes"
"persistentvolumeclaims/kubelet_volume_stats_inodes"
"namespaces/kube_pod_container_status_terminated"
"pods/kube_pod_container_resource_requests_cpu_cores"
"pods/kube_pod_container_status_running"
# 安装grafana
[root@node1 ~]# kubectl apply -f grafana/grafana.yaml 
deployment.apps/monitoring-grafana created
service/monitoring-grafana created

附件列表

• prometheus.tar.gz