28、手动部署kubernetes集群

# 配置主机名解析
[root@node01 ~]# cat /etc/hosts|grep '.*com$'
172.16.1.1 node1 node01 node01.cce.com etcd01 etcd01.cce.com
172.16.1.2 node2 node02 node02.cce.com etcd02 etcd02.cce.com
172.16.1.3 node3 node03 node03.cce.com etcd03 etcd03.cce.com
# 配置yum源(在三个节点都执行)
[root@node01 ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
# 分别安装etcd(在三个节点都执行)
[root@node01 ~]# yum install -y etcd
# 创建数据目录(在三个节点都执行)
[root@node01 ~]# mkdir -pv /data/etcd/data
[root@node01 ~]# chown etcd:etcd /data/etcd/ -R
# 配置etcd(在三个节点都配置)
[root@node01 ~]# grep '^[^#]' /etc/etcd/etcd.conf
ETCD_DATA_DIR="/data/etcd/data" # 数据目录
ETCD_LISTEN_PEER_URLS="http://172.16.1.1:2380" # 集群节点内部通信使用的URL
ETCD_LISTEN_CLIENT_URLS="http://172.16.1.1:2379" # 供外部客户端使用的url
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380" # 广播给集群内其他成员访问的UR
ETCD_ADVERTISE_CLIENT_URLS="http://172.16.1.1:2379" # 广播给外部客户端使用的url，和ETCD_LISTEN_CLIENT_URLS一样
ETCD_INITIAL_CLUSTER="etcd01=http://172.16.1.1:2380,etcd02=http://172.16.1.2:2380,etcd03=http://172.16.1.3:2380" # 集群成员初始化
# 启动etcd(在三个节点都执行)
[root@node01 ~]# systemctl daemon-reload
[root@node01 ~]# systemctl start etcd
# 查看集群选举信息
[root@node01 ~]# etcdctl --endpoints="http://172.16.1.1:2379" member list
23f7d853639f3654: name=etcd03 peerURLs=http://172.16.1.3:2380 clientURLs=http://172.16.1.3:2379 isLeader=false
821e51093cc43832: name=etcd01 peerURLs=http://172.16.1.1:2380 clientURLs=http://172.16.1.1:2379 isLeader=false
9f72c56430a62979: name=etcd02 peerURLs=http://172.16.1.2:2380 clientURLs=http://172.16.1.2:2379 isLeader=true
# 查看集群状态
[root@node01 ~]# etcdctl --endpoints="http://172.16.1.1:2379" cluster-health
member 23f7d853639f3654 is healthy: got healthy result from http://172.16.1.3:2379
member 821e51093cc43832 is healthy: got healthy result from http://172.16.1.1:2379
member 9f72c56430a62979 is healthy: got healthy result from http://172.16.1.2:2379
cluster is healthy

# 生成etcd证书
[root@node01 ~]# git clone https://github.com/iKubernetes/k8s-certs-generator.git
[root@node01 ~]# cd k8s-certs-generator/
[root@node01 k8s-certs-generator]# bash gencerts.sh etcd
[root@node01 k8s-certs-generator]# ls etcd/pki/
apiserver-etcd-client.crt ca.crt client.crt peer.crt server.crt
apiserver-etcd-client.key ca.key client.key peer.key server.key
[root@node01 k8s-certs-generator]# cp -a etcd/pki/ /etc/etcd/pki
# 分发给各节点
[root@node01 k8s-certs-generator]# scp -r /etc/etcd/pki node2:/etc/etcd/pki
[root@node01 k8s-certs-generator]# scp -r /etc/etcd/pki node3:/etc/etcd/pki
# 分别修改三台etcd配置Security配置段，并且链接全部变成https，既然启用了ssl，那么就不能再使用IP了，得使用域名了
[root@node01 ~]# grep '^[^#]' /etc/etcd/etcd.conf
ETCD_DATA_DIR="/data/etcd/data"
ETCD_LISTEN_PEER_URLS="https://172.16.1.1:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.1.1:2379"
ETCD_NAME="etcd01"
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://etcd01.cce.com:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://etcd01.cce.com:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://etcd01.cce.com:2380,etcd02=https://node02.cce.com:2380,etcd03=https://node03.cce.com:2380"  # 这里证书使用的域名cce.com，所以这里集群内部访问也需要使用域名，否则会报错error "remote error: tls: bad certificate", ServerName
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_CERT_FILE="/etc/etcd/pki/server.crt"
ETCD_KEY_FILE="/etc/etcd/pki/server.key"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/etcd/pki/ca.crt"
ETCD_AUTO_TLS="false"
ETCD_PEER_CERT_FILE="/etc/etcd/pki/peer.crt"
ETCD_PEER_KEY_FILE="/etc/etcd/pki/peer.key"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/pki/ca.crt"
ETCD_PEER_AUTO_TLS="false"
# 重启服务(在三个节点都执行)
[root@node01 ~]# systemctl daemon-reload
[root@node01 ~]# systemctl start etcd
# 检查集群成员信息
[root@node01 ~]# etcdctl --endpoints="https://etcd01.cce.com:2379" --key-file=/etc/etcd/pki/client.key --cert-file=/etc/etcd/pki/client.crt --ca-file=/etc/etcd/pki/ca.crt member list
6147711da86b8a61: name=etcd02 peerURLs=https://node02.cce.com:2380 clientURLs=https://etcd02.cce.com:2379 isLeader=false
88a77e250221767b: name=etcd01 peerURLs=https://node01.cce.com:2380 clientURLs=https://etcd01.cce.com:2379 isLeader=true
97ae5618cec09aaf: name=etcd03 peerURLs=https://node03.cce.com:2380 clientURLs=https://etcd03.cce.com:2379 isLeader=false
# 查看集群状态
[root@node01 ~]# etcdctl --endpoints="https://etcd01.cce.com:2379" --key-file=/etc/etcd/pki/client.key --cert-file=/etc/etcd/pki/client.crt --ca-file=/etc/etcd/pki/ca.crt cluster-health
member 6147711da86b8a61 is healthy: got healthy result from https://etcd02.cce.com:2379
member 88a77e250221767b is healthy: got healthy result from https://etcd01.cce.com:2379
member 97ae5618cec09aaf is healthy: got healthy result from https://etcd03.cce.com:2379
cluster is healthy

# 生成必要的证书和密钥
[root@node01 k8s-certs-generator]# sh gencerts.sh k8s
Enter Domain Name [ilinux.io]: cce.com
Enter Kubernetes Cluster Name [kubernetes]: kubernetes
Enter the IP Address in default namespace 
  of the Kubernetes API Server[10.96.0.1]: 
Enter Master servers name[master01 master02 master03]: node01 node02 node03
# 三台Master都创建木兰路
[root@node01 ~]#  mkdir /etc/kubernetes
# 复制证书
[root@node01 ~]# cp k8s-certs-generator/kubernetes/node01/* /etc/kubernetes/ -r
# 复制node02和node03证书
[root@node01 ~]# scp -r k8s-certs-generator/kubernetes/node02/* node2:/etc/kubernetes/
[root@node01 ~]# scp -r k8s-certs-generator/kubernetes/node03/* node3:/etc/kubernetes/
# 解压kubernetes
[root@node01 ~]# tar xf kubernetes-server-linux-amd64.tar.gz
[root@node01 ~]# mv kubernetes /usr/local/

# 准备Master所需要的二进制文件、Unit文件(直接在github上克隆)
[root@node01 ~]# git clone https://github.com/caichangen/k8s-bin-inst.git
# 复制配置文件
[root@node01 ~]# cp k8s-bin-inst/master/etc/kubernetes/* /etc/kubernetes/
# 复制Unit-file文件
[root@node01 ~]# cp k8s-bin-inst/master/unit-files/* /usr/lib/systemd/system/
# 修改apiserver配置文件
[root@node01 ~]# cat /etc/kubernetes/apiserver
###
# kubernetes system config
#
# The following values are used to configure the kube-apiserver
#

# The address on the local server to listen to.
KUBE_API_ADDRESS="--advertise-address=0.0.0.0"

# The port on the local server to listen on.
KUBE_API_PORT="--secure-port=6443 --insecure-port=0"

# Comma separated list of nodes in the etcd cluster
KUBE_ETCD_SERVERS="--etcd-servers=https://etcd01.cce.com:2379,https://etcd02.cce.com:2379,https://etcd03.cce.com:2379"

# Address range to use for services
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.96.0.0/12"

# default admission control policies
KUBE_ADMISSION_CONTROL="--enable-admission-plugins=NodeRestriction"

# Add your own!
KUBE_API_ARGS="--authorization-mode=Node,RBAC \
    --client-ca-file=/etc/kubernetes/pki/ca.crt \
    --enable-bootstrap-token-auth=true \
    --etcd-cafile=/etc/etcd/pki/ca.crt \
    --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt \
    --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key \
    --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt \
    --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key \
    --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
    --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt \
    --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key \
    --requestheader-allowed-names=front-proxy-client \
    --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt \
    --requestheader-extra-headers-prefix=X-Remote-Extra- \
    --requestheader-group-headers=X-Remote-Group \
    --requestheader-username-headers=X-Remote-User\
    --service-account-key-file=/etc/kubernetes/pki/sa.pub \
    --tls-cert-file=/etc/kubernetes/pki/apiserver.crt \
    --tls-private-key-file=/etc/kubernetes/pki/apiserver.key \
    --token-auth-file=/etc/kubernetes/token.csv"
# 创建运行用户kube
[root@node01 ~]# useradd -r kube
# 创建kubernetes自己用到的临时状态保存目录
[root@node01 ~]# mkdir /var/run/kubernetes
[root@node01 ~]# chown kube:kube /var/run/kubernetes -R
# 启动apiserver服务
[root@node01 ~]# systemctl daemon-reload
[root@node01 ~]# systemctl start kube-apiserver
[root@node01 ~]# netstat -ntlp|grep 6443
tcp6       0      0 :::6443                 :::*                    LISTEN      25829/kube-apiserve 

# 配置kubectl
[root@node01 ~]# ln -sv /usr/local/kubernetes/server/bin/kubectl /usr/bin/kubectl
# 配置自动补齐
[root@node01 ~]# kubectl completion bash > /usr/share/bash-completion/completions/kubectl
[root@node01 ~]# source /usr/share/bash-completion/completions/kubectl
# 配置认证信息
[root@node01 ~]# mkdir ~/.kube
[root@node01 ~]# cp /etc/kubernetes/auth/admin.conf ~/.kube/config
[root@node01 ~]# chown $(id -u):$(id -g) $HOME/.kube/config
# 查看配置
[root@node01 ~]# kubectl config view
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://node01.cce.com:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: k8s-admin
  name: k8s-admin@kubernetes
current-context: k8s-admin@kubernetes
kind: Config
preferences: {}
users:
- name: k8s-admin
  user:
    client-certificate-data: REDACTED
    client-key-data: REDACTED

# 绑用户
[root@node01 ~]# kubectl create clusterrolebinding system:bootstrapper --group=system:bootstrappers --clusterrole=system:node-bootstrapper

# 或者（可以绑，用户，也可以绑租）

# 绑定组
[root@node01 ~]# kubectl create clusterrolebinding system:bootstrapper --user=system:bootstrapper --clusterrole=system:node-bootstrapper
[root@node01 ~]# kubectl describe clusterrolebindings.rbac.authorization.k8s.io system:bootstrapper
Name:         system:bootstrapper
Labels:       <none>
Annotations:  <none>
Role:
  Kind:  ClusterRole
  Name:  system:node-bootstrapper
Subjects:
  Kind  Name                 Namespace
  ----  ----                 ---------
  User  system:bootstrapper 

# 直接启动即可
[root@node01 ~]# systemctl start kube-controller-manager.service

# 直接启动即可
[root@node01 ~]# systemctl start kube-scheduler.service

[root@node01 ~]# kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
etcd-1               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}   
etcd-0               Healthy   {"health":"true"}   
controller-manager   Healthy   ok 

# Docker
[root@node01 ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@node01 ~]# wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker.repo
[root@node04 ~]# yum install -y bash-completion docker-ce
[root@node04 ~]# systemctl start docker
# 配置内核参数
[root@node04 ~]# cat >> /etc/sysctl.conf << EOF
# iptables透明网桥的实现
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
EOF
[root@node04 ~]# sysctl -p
# 设定镜像加速器
[root@node04 ~]# cat >> /etc/docker/daemon.json << EOF
{
    "registry-mirrors": ["https://owcyje1z.mirror.aliyuncs.com"]
}
EOF
# 设定iptables
[root@node04 ~]# sed -i '/^ExecStart.*/aExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT' /usr/lib/systemd/system/docker.service
[root@node04 ~]# systemctl daemon-reload 
[root@node04 ~]# systemctl restart docker

[root@node04 ~]# git clone https://github.com/caichangen/k8s-bin-inst.git 
[root@node04 ~]# cat k8s-bin-inst/nodes/var/lib/kubelet/config.yaml 
address: 0.0.0.0
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: cgroupfs
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
configMapAndSecretChangeDetectionStrategy: Watch
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuCFSQuotaPeriod: 100ms
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: false
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kind: KubeletConfiguration
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeLeaseDurationSeconds: 40
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
port: 10250
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s

# 配置二进制程序
[root@node04 ~]# tar xf kubernetes-node-linux-amd64.tar.gz -C /usr/local/
[root@node04 ~]# ls /usr/local/kubernetes/node/bin/
kubeadm  kubectl  kubelet  kube-proxy
# 配置工作目录
[root@node04 ~]# cp -r k8s-bin-inst/nodes/var/lib/kubelet /var/lib/
[root@node04 ~]# ls /var/lib/kubelet/config.yaml
/var/lib/kubelet/config.yaml
# 提供配置文件
[root@node04 ~]# cp -r k8s-bin-inst/nodes/etc/kubernetes/ /etc/
[root@node04 ~]# ls /etc/kubernetes/
config  kubelet  proxy
# 修改kubelet配置，加入pause镜像加载地址--pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.1，pause镜像很关键，它是用来运行Pod的一个基础架构的镜像，所以必须能拉取到；
[root@node04 ~]# grep 'pause' /etc/kubernetes/kubelet
    --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.1 \
# 将先前部署Master生成的证书COPY过来
[root@node01 ~]# scp -r k8s-certs-generator/kubernetes/kubelet/* node4:/etc/kubernetes/
[root@node04 ~]# mkdir /etc/kubernetes/manifests
[root@node04 ~]# ls /etc/kubernetes/{auth,pki,manifests} -d
/etc/kubernetes/auth  /etc/kubernetes/manifests  /etc/kubernetes/pki
# 可以看到我们的/etc/kubernetes/kubelet配置文件，里面需要用到CNI插件，所以我们需要在github上去下载CNI插件到本地，下载地址<a href="https://github.com/containernetworking/plugins/releases">https://github.com/containernetworking/plugins/releases</a>
[root@node04 ~]# mkdir -p /opt/cni/bin
[root@node04 ~]# tar xf cni-plugins-linux-amd64-v0.8.5.tgz -C /opt/cni/bin/
[root@node04 ~]# ls /opt/cni/bin/
bandwidth  dhcp      flannel      host-local  loopback  portmap  sbr     tuning
bridge     firewall  host-device  ipvlan      macvlan   ptp      static  vlan
# 复制kubelet的unit-file文件
[root@node04 ~]# cp k8s-bin-inst/nodes/unit-files/kubelet.service /usr/lib/systemd/system/
启动kubelet
[root@node04 ~]# systemctl daemon-reload 
[root@node04 ~]# systemctl start kubelet 

# 查看主节点是否收到kubelet启动之后发出的证书签署请求
[root@node01 ~]# kubectl get csr
NAME        AGE   REQUESTOR             CONDITION
csr-vcrlc   26s   system:bootstrapper   Pending
# 签署证书
[root@node01 ~]# kubectl certificate approve csr-vcrlc
certificatesigningrequest.certificates.k8s.io/csr-vcrlc approved
# 查看加入进来的Node
[root@node01 ~]# kubectl get nodes -o wide
NAME             STATUS     ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION          CONTAINER-RUNTIME
node04.cce.com   NotReady   <none>   89s   v1.17.3   172.16.1.4    <none>        CentOS Linux 7 (Core)   3.10.0-957.el7.x86_64   docker://19.3.7

# unit-file环境变量配置文件
[root@node04 ~]# ls /etc/kubernetes/proxy
/etc/kubernetes/proxy
# 复制kube-proxy配置
[root@node01 ~]# scp -r k8s-bin-inst/nodes/var/lib/kube-proxy/ node4:/var/lib/
[root@node04 ~]# ls /var/lib/kube-proxy/
config.yaml
# 启用ipvs模块
[root@node04 ~]# cat >> /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
#
ipvs_mods_dir="/usr/lib/modules/\$(uname -r)/kernel/net/netfilter/ipvs"
for mod in \$(ls \$ipvs_mods_dir | grep -o "^[^.]*"); do
    /sbin/modinfo -F filename \$mod &> /dev/null
    if [ \$? -eq 0 ]; then
        /sbin/modprobe \$mod
    fi
done
EOF
# 修改文件权限，并手动为当前系统加载内核模块
[root@node1 ~]# chmod +x /etc/sysconfig/modules/ipvs.modules
[root@node1 ~]# bash /etc/sysconfig/modules/ipvs.modules
# 安装ipvsadm管理工具
[root@node04 ~]# yum install -y ipvsadm
# 复制unit-file文件
[root@node01 ~]# scp k8s-bin-inst/nodes/unit-files/kube-proxy.service node4:/usr/lib/systemd/system
# 启动kube-proxy
[root@node04 ~]# systemctl daemon-reload                  
[root@node04 ~]# systemctl start kube-proxy.service

# 下载配置清单
[root@node01 ~]# wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
# 修改网络模式为host-gw(测试flannel的vxlan模式对于kubernetes 1.17版本的支持貌似不是很好)
[root@node01 ~]# grep 'host-gw' kube-flannel.yml 
        "Type": "host-gw"
# 修改镜像为国内镜像，拉取快
[root@node01 ~]# sed -i 's@\<quay.io@quay.mirrors.ustc.edu.cn@g' kube-flannel.yml
# 配置flannel网络并查看nodes状态
[root@node01 ~]# kubectl apply -f kube-flannel.yml 
[root@node01 ~]# kubectl get nodes
NAME             STATUS   ROLES    AGE   VERSION
node04.cce.com   Ready    <none>   40m   v1.17.3

# 复制程序包
[root@node01 ~]# scp kubernetes-server-linux-amd64.tar.gz node2:/root
# 解压二进制程序
[root@node02 ~]# tar xf kubernetes-server-linux-amd64.tar.gz -C /usr/local/
# 复制配置文件
[root@node01 ~]# scp k8s-bin-inst/master/etc/kubernetes/* node2:/etc/kubernetes/
# 查看此前复制的证书和配置文件
[root@node02 ~]# ll /etc/kubernetes/
total 24
-rw-r--r-- 1 root root 1929 Mar  9 09:24 apiserver
drwxr-xr-x 2 root root   77 Mar  9 00:05 auth
-rw-r--r-- 1 root root  538 Mar  9 09:24 config
-rw-r--r-- 1 root root 1018 Mar  9 09:24 controller-manager
drwxr-xr-x 2 root root 4096 Mar  9 00:05 pki
-rw-r--r-- 1 root root  211 Mar  9 09:24 scheduler
-rw-r--r-- 1 root root   75 Mar  9 00:05 token.csv
# 复制unit-file
[root@node01 ~]# scp k8s-bin-inst/master/unit-files/* node2:/usr/lib/systemd/system
# 修改apiserver的etcd的配置
[root@node02 ~]# grep 'etcd0' /etc/kubernetes/apiserver
KUBE_ETCD_SERVERS="--etcd-servers=https://etcd01.cce.com:2379,https://etcd02.cce.com:2379,https://etcd03.cce.com:2379"
# 创建运行用户
[root@node02 ~]# useradd -r kube
# 创建工作目录
[root@node02 ~]# mkdir /var/lib/kubernetes
[root@node02 ~]# chown kube:kube /var/lib/kubernetes
# 启动服务
[root@node02 ~]# systemctl daemon-reload
[root@node02 ~]# systemctl start kube-apiserver.service 
[root@node02 ~]# netstat -ntlp|grep 6443
tcp6       0      0 :::6443                 :::*                    LISTEN      6947/kube-apiserver 

# 配置kubectl
[root@node02 ~]# ln -sv /usr/local/kubernetes/server/bin/kubectl /usr/bin/kubectl
# 配置自动补齐
[root@node02 ~]# kubectl completion bash > /usr/share/bash-completion/completions/kubectl
[root@node02 ~]# source /usr/share/bash-completion/completions/kubectl
# 配置认证信息
[root@node02 ~]# mkdir ~/.kube
[root@node02 ~]# cp /etc/kubernetes/auth/admin.conf ~/.kube/config
[root@node02 ~]# chown $(id -u):$(id -g) $HOME/.kube/config
[root@node02 ~]# kubectl get nodes
NAME             STATUS   ROLES    AGE   VERSION
node04.cce.com   Ready    <none>   8h    v1.17.3

[root@node02 ~]# systemctl start kube-scheduler.service kube-controller-manager.service

# 复制程序包
[root@node01 ~]# scp kubernetes-server-linux-amd64.tar.gz node3:/root
# 解压二进制程序
[root@node03 ~]# tar xf kubernetes-server-linux-amd64.tar.gz -C /usr/local/
# 复制配置文件
[root@node01 ~]# scp k8s-bin-inst/master/etc/kubernetes/* node3:/etc/kubernetes/
# 查看此前复制的证书和配置文件
[root@node03 ~]# ll /etc/kubernetes/
total 24
-rw-r--r-- 1 root root 1929 Mar  9 10:12 apiserver
drwxr-xr-x 2 root root   77 Mar  9 00:05 auth
-rw-r--r-- 1 root root  538 Mar  9 10:12 config
-rw-r--r-- 1 root root 1018 Mar  9 10:12 controller-manager
drwxr-xr-x 2 root root 4096 Mar  9 00:05 pki
-rw-r--r-- 1 root root  211 Mar  9 10:12 scheduler
-rw-r--r-- 1 root root   75 Mar  9 00:05 token.csv
# 复制unit-file
[root@node01 ~]# scp k8s-bin-inst/master/unit-files/* node3:/usr/lib/systemd/system
# 修改apiserver的etcd的配置
[root@node03 ~]# grep 'etcd0' /etc/kubernetes/apiserver
KUBE_ETCD_SERVERS="--etcd-servers=https://etcd01.cce.com:2379,https://etcd02.cce.com:2379,https://etcd03.cce.com:2379"
# 创建运行用户
[root@node03 ~]# useradd -r kube
# 创建工作目录
[root@node03 ~]# mkdir /var/lib/kubernetes
[root@node03 ~]# chown kube:kube /var/lib/kubernetes
# 启动服务
[root@node03 ~]# systemctl daemon-reload
[root@node03 ~]# systemctl start kube-apiserver.service 
[root@node03 ~]# netstat -ntlp|grep 6443
tcp6       0      0 :::6443                 :::*                    LISTEN      6947/kube-apiserver 

# 配置kubectl
[root@node03 ~]# ln -sv /usr/local/kubernetes/server/bin/kubectl /usr/bin/kubectl
# 配置自动补齐
[root@node03 ~]# kubectl completion bash > /usr/share/bash-completion/completions/kubectl
[root@node03 ~]# source /usr/share/bash-completion/completions/kubectl
# 配置认证信息
[root@node03 ~]# mkdir ~/.kube
[root@node03 ~]# cp /etc/kubernetes/auth/admin.conf ~/.kube/config
[root@node03 ~]# chown $(id -u):$(id -g) $HOME/.kube/config
[root@node03 ~]# kubectl get nodes
NAME             STATUS   ROLES    AGE   VERSION
node04.cce.com   Ready    <none>   8h    v1.17.3

[root@node03 ~]# systemctl start kube-scheduler.service kube-controller-manager.service

# 创建目录
[root@node01 ~]# mkdir /etc/kubernetes/coredns
[root@node01 ~]# cd /etc/kubernetes/coredns
# 下载文件
[root@node01 coredns]# wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed
# 下载处理脚本
[root@node01 coredns]# wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/deploy.sh
# coredns.yaml.sed是一个模版文件，我们需要执行脚本，带入自定义dns的ip，生成一个coredns的YAML格式的配置清单
[root@node01 coredns]# bash deploy.sh -i 10.96.0.10 -r "10.96.0.0/12" -s -t coredns.yaml.sed > coredns.yaml
# 创建coredns
[root@node01 coredns]# kubectl apply -f coredns.yaml
[root@node02 ~]# kubectl get pods -n kube-system                              
NAME                          READY   STATUS    RESTARTS   AGE
coredns-59845f77f8-kcfqh      1/1     Running   0          44s
kube-flannel-ds-amd64-4gf4d   1/1     Running   1          9h
# 创建一个Pod测试解析，检测coredns工作是否正常
[root@node02 ~]# cat >> busybox.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF
[root@node02 ~]# kubectl apply -f busybox.yaml
# 查看busybox容器内部的resolv.conf内容，测试正常
[root@node02 ~]# kubectl exec busybox cat /etc/resolv.conf 
nameserver 10.96.0.10
search default.svc.cluster.local svc.cluster.local cluster.local lan cce.com
options ndots:5
# 在busybox 的pod里解析kubernetes.default 的IP地址，测试正常
[root@node02 ~]# kubectl exec -ti busybox -- nslookup kubernetes.default
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes.default
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local

# Docker
[root@node01 ~]# wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@node01 ~]# wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker.repo
[root@node05 ~]# yum install -y bash-completion docker-ce
[root@node05 ~]# systemctl start docker
# 配置内核参数
[root@node05 ~]# cat >> /etc/sysctl.conf << EOF
# iptables透明网桥的实现
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-arptables = 1
EOF
[root@node05 ~]# sysctl -p
# 设定镜像加速器
[root@node05 ~]# cat >> /etc/docker/daemon.json << EOF
{
    "registry-mirrors": ["https://owcyje1z.mirror.aliyuncs.com"]
}
EOF
# 设定iptables
[root@node05 ~]# sed -i '/^ExecStart.*/aExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT' /usr/lib/systemd/system/docker.service
[root@node05 ~]# systemctl daemon-reload 
[root@node05 ~]# systemctl restart docker

# 配置二进制程序
[root@node05 ~]# tar xf kubernetes-node-linux-amd64.tar.gz -C /usr/local/
[root@node05 ~]# ls /usr/local/kubernetes/node/bin/
kubeadm kubectl kubelet kube-proxy
# 配置工作目录
[root@node01 ~]# scp -r k8s-bin-inst/nodes/var/lib/kubelet node5:/var/lib/ 
[root@node05 ~]# ls /var/lib/kubelet/config.yaml
/var/lib/kubelet/config.yaml
# 提供配置文件
[root@node05 ~]# scp -r k8s-bin-inst/nodes/etc/kubernetes node5:/etc/
[root@node05 ~]# ls /etc/kubernetes/
config  kubelet  proxy
# 修改kubelet配置，加入pause镜像加载地址--pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.1，pause镜像很关键，它是用来运行Pod的一个基础架构的镜像，所以必须能拉取到；
[root@node05 ~]# grep 'pause' /etc/kubernetes/kubelet
    --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.1 \
# 将先前部署Master生成的证书COPY过来
[root@node01 ~]# scp -r k8s-certs-generator/kubernetes/kubelet/* node5:/etc/kubernetes/
[root@node05 ~]# mkdir /etc/kubernetes/manifests
[root@node05 ~]# ls /etc/kubernetes/{auth,pki,manifests} -d
/etc/kubernetes/auth  /etc/kubernetes/manifests  /etc/kubernetes/pki
# 可以看到我们的/etc/kubernetes/kubelet配置文件，里面需要用到CNI插件，所以我们需要在github上去下载CNI插件到本地，下载地址https://github.com/containernetworking/plugins/releases
[root@node05 ~]# mkdir -p /opt/cni/bin
[root@node05 ~]# tar xf cni-plugins-linux-amd64-v0.8.5.tgz -C /opt/cni/bin/
[root@node05 ~]# ls /opt/cni/bin/
bandwidth dhcp flannel host-local loopback portmap sbr tuning
bridge firewall host-device ipvlan macvlan ptp static vlan
# 复制kubelet的unit-file文件
[root@node05 ~]# scp k8s-bin-inst/nodes/unit-files/kubelet.service node5:/usr/lib/systemd/system/
启动kubelet
[root@node05 ~]# systemctl daemon-reload 
[root@node05 ~]# systemctl start kubelet 

# 查看主节点是否收到kubelet启动之后发出的证书签署请求
[root@node01 ~]# kubectl get csr -n kube-system 
NAME        AGE   REQUESTOR             CONDITION
csr-q89z4   3s    system:bootstrapper   Pending
# 签署证书
[root@node01 ~]# kubectl certificate approve csr-q89z4
certificatesigningrequest.certificates.k8s.io/csr-q89z4 approved
# 查看加入进来的Node
[root@node01 ~]# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
node04.cce.com NotReady <none> 89s v1.17.3 172.16.1.4 <none> CentOS Linux 7 (Core) 3.10.0-957.el7.x86_64 docker://19.3.7

# unit-file环境变量配置文件
[root@node04 ~]# ls /etc/kubernetes/proxy
/etc/kubernetes/proxy
# 复制kube-proxy配置
[root@node01 ~]# scp -r k8s-bin-inst/nodes/var/lib/kube-proxy/ node5:/var/lib/
[root@node05 ~]# ls /var/lib/kube-proxy/
config.yaml
# 启用ipvs模块
[root@node05 ~]# cat >> /etc/sysconfig/modules/ipvs.modules << EOF
#!/bin/bash
#
ipvs_mods_dir="/usr/lib/modules/\$(uname -r)/kernel/net/netfilter/ipvs"
for mod in \$(ls \$ipvs_mods_dir | grep -o "^[^.]*"); do
    /sbin/modinfo -F filename \$mod &> /dev/null
    if [ \$? -eq 0 ]; then
        /sbin/modprobe \$mod
    fi
done
EOF
# 修改文件权限，并手动为当前系统加载内核模块
[root@node1 ~]# chmod +x /etc/sysconfig/modules/ipvs.modules
[root@node1 ~]# bash /etc/sysconfig/modules/ipvs.modules
# 安装ipvsadm管理工具
[root@node05 ~]# yum install -y ipvsadm
# 复制unit-file文件
[root@node01 ~]# scp k8s-bin-inst/nodes/unit-files/kube-proxy.service node5:/usr/lib/systemd/system
# 启动kube-proxy
[root@node05 ~]# systemctl daemon-reload                  
[root@node05 ~]# systemctl start kube-proxy.service
# 查看节点
[root@node01 ~]# kubectl get pods -n kube-system
NAME                          READY   STATUS    RESTARTS   AGE
coredns-59845f77f8-kcfqh      1/1     Running   0          122m
kube-flannel-ds-amd64-54r8z   1/1     Running   0          2m21s
kube-flannel-ds-amd64-q9t9p   1/1     Running   0          14m

# 查看现有的endpoint(独占锁)
[root@node01 ~]# kubectl get endpoints -n kube-system
NAME                      ENDPOINTS   AGE
kube-controller-manager   <none>      10h
kube-scheduler            <none>      10h
# 查看kube-controller-manager现在是谁抢占的
[root@node01 ~]# kubectl get endpoints -n kube-system kube-controller-manager -o yaml # 可以看到此时是我们的node1抢占了kube-controller-manager这个endpoint
apiVersion: v1
kind: Endpoints
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"node01.cce.com_10e023e6-eae3-4b68-b562-ada190830854","leaseDurationSeconds":15,"acquireTime":"2020-03-09T01:17:58Z","renewTime":"2020-03-09T02:23:09Z","leaderTransitions":1}'
  creationTimestamp: "2020-03-08T16:20:41Z"
  name: kube-controller-manager
  namespace: kube-system
  resourceVersion: "19492"
  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
  uid: 52ea0caa-a6b5-47a8-93ec-a0fa581f7a0a
# holderIdentity：持有人；
# leaseDurationSeconds：租赁期限；
# acquireTime：获取时间(时间差8个小时)；
# renewTime：更新时间(时间差8个小时)；
# leaderTransitions：leader切换次数；

# 测试将node1的kube-controller-manager停掉
[root@node01 ~]# systemctl stop kube-controller-manager.service 
# 查看其他Master节点是否会抢占kube-controller-manager的这个endpoint
[root@node01 ~]# kubectl get endpoints -n kube-system kube-controller-manager -o yaml # 可以看到已经切换到了node2
apiVersion: v1
kind: Endpoints
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"node02.cce.com_6d9408f8-2bb1-4a71-a82f-5a75b955e5e2","leaseDurationSeconds":15,"acquireTime":"2020-03-09T02:27:42Z","renewTime":"2020-03-09T02:29:06Z","leaderTransitions":2}'
  creationTimestamp: "2020-03-08T16:20:41Z"
  name: kube-controller-manager
  namespace: kube-system
  resourceVersion: "20464"
  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
  uid: 52ea0caa-a6b5-47a8-93ec-a0fa581f7a0a

[root@node01 ~]# cat test.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    author: caichangen
  name: cce
spec:
  replicas: 4
  selector:
    matchLabels:
      app: cce
  template:
    metadata:
      name: ccepod
      labels:
        app: cce
    spec:
      restartPolicy: Always
      containers:
      - image: ikubernetes/myapp:v1
        name: ccecontainers
---
apiVersion: v1
kind: Service
metadata:
  name: cce
  labels:
    app: cce
spec:
  type: "NodePort"
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
    nodePort: 30080
  selector:
    app: cce
[root@node01 ~]# kubectl apply -f test.yaml
[root@node01 ~]# kubectl apply -f test.yaml 
deployment.apps/cce created
service/cce created
[root@node01 ~]# kubectl get pod,svc -o wide -l app=cce
NAME                       READY   STATUS    RESTARTS   AGE   IP           NODE             NOMINATED NODE   READINESS GATES
pod/cce-78cd5bfc9f-566l6   1/1     Running   0          4s    10.244.0.8   node04.cce.com   <none>           <none>
pod/cce-78cd5bfc9f-h8g6v   1/1     Running   0          4s    10.244.1.4   node05.cce.com   <none>           <none>
pod/cce-78cd5bfc9f-txcpd   1/1     Running   0          4s    10.244.0.7   node04.cce.com   <none>           <none>
pod/cce-78cd5bfc9f-vrp6g   1/1     Running   0          4s    10.244.1.5   node05.cce.com   <none>           <none>

NAME          TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)        AGE   SELECTOR
service/cce   NodePort   10.98.231.67   <none>        80:30080/TCP   4s    app=cce
# 在有kube-proxy组件的节点查看ipvs规则
[root@node04 ~]# ipvsadm -L
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  localhost:30080 rr
  -> 10.244.0.7:http              Masq    1      0          0         
  -> 10.244.0.8:http              Masq    1      0          0         
  -> 10.244.1.4:http              Masq    1      0          0         
  -> 10.244.1.5:http              Masq    1      0          0   
# 在有kube-proxy组件的测试访问svc地址
[root@node04 ~]# curl 10.98.231.67/hostname.html
cce-78cd5bfc9f-566l6
[root@node04 ~]# curl 10.98.231.67/hostname.html
cce-78cd5bfc9f-vrp6g
[root@node04 ~]# curl 10.98.231.67/hostname.html
cce-78cd5bfc9f-txcpd
[root@node04 ~]# curl 10.98.231.67/hostname.html
cce-78cd5bfc9f-h8g6v
#  在有kube-proxy组件的节点地址(该deployment是NodePort的)
[root@node04 ~]# curl 172.16.1.4:30080/hostname.html
cce-78cd5bfc9f-h8g6v
[root@node04 ~]# curl 172.16.1.4:30080/hostname.html
cce-78cd5bfc9f-566l6
[root@node04 ~]# curl 172.16.1.4:30080/hostname.html
cce-78cd5bfc9f-vrp6g
[root@node04 ~]# curl 172.16.1.4:30080/hostname.html
cce-78cd5bfc9f-txcpd

[root@node01 ~]# wget http://nginx.org/download/nginx-1.16.1.tar.gz
[root@node01 ~]# tar xf nginx-1.16.1.tar.gz 
[root@node01 ~]# cd nginx-1.16.1/
[root@node01 ~]# useradd -r www 
[root@node01 ~]# ./configure --prefix=/usr/local/nginx \
--prefix=/usr/local/nginx \
--with-pcre --with-file-aio \
--with-http_ssl_module \
--with-http_flv_module \
--with-http_dav_module \
--with-http_sub_module \
--with-http_mp4_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_gzip_static_module \
--with-http_stub_status_module \
--lock-path=/usr/local/nginx/nginx.lock \
--pid-path=/var/run/nginx.pid \
--http-scgi-temp-path=/usr/local/nginx/scgi \
--http-fastcgi-temp-path=/usr/local/nginx/fcgi/ \
--http-uwsgi-temp-path=/usr/local/nginx/uwsgi \
--http-proxy-temp-path=/usr/local/nginx/proxy/ \
--http-client-body-temp-path=/usr/local/nginx/client/ \
--with-stream \
--user=www
[root@node01 nginx-1.16.1]# make -j 2
[root@node01 nginx-1.16.1]# make install
# 配置nginx，注意stream配置段不能放在http内部，因为它不是转发http协议的
[root@node01 ~]# cat >> /usr/local/nginx/conf/nginx.conf << EOF
stream {
      log_format ws "$remote_addr $upstream_addr $time_local $status";
      access_log /var/log/nginx/k8s.log ws;
      server {
          listen 172.16.1.1:8443;
          proxy_pass app_server;
      }
      upstream app_server{
          server 172.16.1.1:6443 max_fails=3 fail_timeout=30;  # Master1
          server 172.16.1.2:6443 max_fails=3 fail_timeout=30;  # Master2
          server 172.16.1.3:6443 max_fails=3 fail_timeout=30;  # Master3
      }
}
EOF
[root@node01 ~]# /usr/local/nginx/sbin/nginx
# 测试访问
[root@node01 ~]# curl https://172.16.1.1:8443 -k # 成功放到到apiserver
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {

  },
  "status": "Failure",
  "message": "forbidden: User \"system:anonymous\" cannot get path \"/\"",
  "reason": "Forbidden",
  "details": {

  },
  "code": 403
}