声明： 如果您有更好的技术与作者分享，或者商业合作； 请访问作者个人网站 http://www.esqabc.com/view/message.html 留言给作者。 如果该案例触犯您的专利，请在这里：http://www.esqabc.com/view/message.html 留言给作者说明原由 作者一经查实，马上删除。
a、kubernetes - master节点运行组件如下:
kube-apiserverkube-schedulerkube-controller-manager如没有特殊说明，一般都在k8s-01服务器操作
前提提条件、服务器，请查看这个地址：https://blog.csdn.net/esqabc/article/details/102726771
a、下载kubernetes二进制包
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# wget http://down.i4t.com/k8s1.14/kubernetes-server-linux-amd64.tar.gz [root@k8s-01 work]# tar -xzvf kubernetes-server-linux-amd64.tar.gz [root@k8s-01 work]# cd kubernetes [root@k8s-01 kubernetes]# tar -xzvf kubernetes-src.tar.gz
b、分发到所有master节点
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# source /opt/k8s/bin/environment.sh
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kubernetes/server/bin/{apiextensions-apiserver,cloud-controller-manager,kube-apiserver,kube-controller-manager,kube-proxy,kube-scheduler,kubeadm,kubectl,kubelet,mounter} root@${node_ip}:/opt/k8s/bin/ ssh root@${node_ip} "chmod +x /opt/k8s/bin/*" donec、创建Kubernetes 证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# cat > kubernetes-csr.json <<EOF 添加下面内容：
{ "CN": "kubernetes", "hosts": [ "127.0.0.1", "172.26.16.249", "172.26.16.250", "172.26.16.251", "172.26.16.252", "10.254.0.1", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local." ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "k8s", "OU": "4Paradigm" } ] } EOF注意：需要将集群的所有IP都添加进去
d、生成证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \ -ca-key=/opt/k8s/work/ca-key.pem \ -config=/opt/k8s/work/ca-config.json \ -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes[root@k8s-01 ~]# ls kubernetes*pem
e、分发到所有master节点
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "mkdir -p /etc/kubernetes/cert" scp kubernetes*.pem root@${node_ip}:/etc/kubernetes/cert/ donef、创建加密配置文件
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > encryption-config.yaml <<EOF 添加下面内容
kind: EncryptionConfig apiVersion: v1 resources: - resources: - secrets providers: - aescbc: keys: - name: key1 secret: ${ENCRYPTION_KEY} - identity: {} EOFg、将加密配置文件拷贝到master节点的/etc/kubernetes目录下
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp encryption-config.yaml root@${node_ip}:/etc/kubernetes/ doneh、创建审计策略文件
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > audit-policy.yaml <<EOF 添加下面内容：
apiVersion: audit.k8s.io/v1beta1 kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, so drop them. - level: None resources: - group: "" resources: - endpoints - services - services/status users: - 'system:kube-proxy' verbs: - watch - level: None resources: - group: "" resources: - nodes - nodes/status userGroups: - 'system:nodes' verbs: - get - level: None namespaces: - kube-system resources: - group: "" resources: - endpoints users: - 'system:kube-controller-manager' - 'system:kube-scheduler' - 'system:serviceaccount:kube-system:endpoint-controller' verbs: - get - update - level: None resources: - group: "" resources: - namespaces - namespaces/status - namespaces/finalize users: - 'system:apiserver' verbs: - get # Don't log HPA fetching metrics. - level: None resources: - group: metrics.k8s.io users: - 'system:kube-controller-manager' verbs: - get - list # Don't log these read-only URLs. - level: None nonResourceURLs: - '/healthz*' - /version - '/swagger*' # Don't log events requests. - level: None resources: - group: "" resources: - events # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes - level: Request omitStages: - RequestReceived resources: - group: "" resources: - nodes/status - pods/status users: - kubelet - 'system:node-problem-detector' - 'system:serviceaccount:kube-system:node-problem-detector' verbs: - update - patch - level: Request omitStages: - RequestReceived resources: - group: "" resources: - nodes/status - pods/status userGroups: - 'system:nodes' verbs: - update - patch # deletecollection calls can be large, don't log responses for expected namespace deletions - level: Request omitStages: - RequestReceived users: - 'system:serviceaccount:kube-system:namespace-controller' verbs: - deletecollection # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, # so only log at the Metadata level. - level: Metadata omitStages: - RequestReceived resources: - group: "" resources: - secrets - configmaps - group: authentication.k8s.io resources: - tokenreviews # Get repsonses can be large; skip them. - level: Request omitStages: - RequestReceived resources: - group: "" - group: admissionregistration.k8s.io - group: apiextensions.k8s.io - group: apiregistration.k8s.io - group: apps - group: authentication.k8s.io - group: authorization.k8s.io - group: autoscaling - group: batch - group: certificates.k8s.io - group: extensions - group: metrics.k8s.io - group: networking.k8s.io - group: policy - group: rbac.authorization.k8s.io - group: scheduling.k8s.io - group: settings.k8s.io - group: storage.k8s.io verbs: - get - list - watch # Default level for known APIs - level: RequestResponse omitStages: - RequestReceived resources: - group: "" - group: admissionregistration.k8s.io - group: apiextensions.k8s.io - group: apiregistration.k8s.io - group: apps - group: authentication.k8s.io - group: authorization.k8s.io - group: autoscaling - group: batch - group: certificates.k8s.io - group: extensions - group: metrics.k8s.io - group: networking.k8s.io - group: policy - group: rbac.authorization.k8s.io - group: scheduling.k8s.io - group: settings.k8s.io - group: storage.k8s.io # Default level for all other requests. - level: Metadata omitStages: - RequestReceived EOFi、分发审计策略文件
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp audit-policy.yaml root@${node_ip}:/etc/kubernetes/audit-policy.yaml donej、创建证书签名请求
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > proxy-client-csr.json <<EOF 添加下面内容：
{ "CN": "aggregator", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "k8s", "OU": "4Paradigm" } ] } EOFk、生成证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/etc/kubernetes/cert/ca.pem \ -ca-key=/etc/kubernetes/cert/ca-key.pem \ -config=/etc/kubernetes/cert/ca-config.json \ -profile=kubernetes proxy-client-csr.json | cfssljson -bare proxy-client[root@k8s-01 ~]# ls proxy-client*.pem
l、将生成的证书和私钥文件分发到master节点
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp proxy-client*.pem root@${node_ip}:/etc/kubernetes/cert/ donem、创建kube-apiserver启动文件
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# cat > kube-apiserver.service.template <<EOF 添加下面内容：
[Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] WorkingDirectory=${K8S_DIR}/kube-apiserver ExecStart=/opt/k8s/bin/kube-apiserver \\ --advertise-address=##NODE_IP## \\ --default-not-ready-toleration-seconds=360 \\ --default-unreachable-toleration-seconds=360 \\ --feature-gates=DynamicAuditing=true \\ --max-mutating-requests-inflight=2000 \\ --max-requests-inflight=4000 \\ --default-watch-cache-size=200 \\ --delete-collection-workers=2 \\ --encryption-provider-config=/etc/kubernetes/encryption-config.yaml \\ --etcd-cafile=/etc/kubernetes/cert/ca.pem \\ --etcd-certfile=/etc/kubernetes/cert/kubernetes.pem \\ --etcd-keyfile=/etc/kubernetes/cert/kubernetes-key.pem \\ --etcd-servers=${ETCD_ENDPOINTS} \\ --bind-address=##NODE_IP## \\ --secure-port=6443 \\ --tls-cert-file=/etc/kubernetes/cert/kubernetes.pem \\ --tls-private-key-file=/etc/kubernetes/cert/kubernetes-key.pem \\ --insecure-port=0 \\ --audit-dynamic-configuration \\ --audit-log-maxage=15 \\ --audit-log-maxbackup=3 \\ --audit-log-maxsize=100 \\ --audit-log-truncate-enabled \\ --audit-log-path=${K8S_DIR}/kube-apiserver/audit.log \\ --audit-policy-file=/etc/kubernetes/audit-policy.yaml \\ --profiling \\ --anonymous-auth=false \\ --client-ca-file=/etc/kubernetes/cert/ca.pem \\ --enable-bootstrap-token-auth \\ --requestheader-allowed-names="aggregator" \\ --requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\ --requestheader-extra-headers-prefix="X-Remote-Extra-" \\ --requestheader-group-headers=X-Remote-Group \\ --requestheader-username-headers=X-Remote-User \\ --service-account-key-file=/etc/kubernetes/cert/ca.pem \\ --authorization-mode=Node,RBAC \\ --runtime-config=api/all=true \\ --enable-admission-plugins=NodeRestriction \\ --allow-privileged=true \\ --apiserver-count=3 \\ --event-ttl=168h \\ --kubelet-certificate-authority=/etc/kubernetes/cert/ca.pem \\ --kubelet-client-certificate=/etc/kubernetes/cert/kubernetes.pem \\ --kubelet-client-key=/etc/kubernetes/cert/kubernetes-key.pem \\ --kubelet-https=true \\ --kubelet-timeout=10s \\ --proxy-client-cert-file=/etc/kubernetes/cert/proxy-client.pem \\ --proxy-client-key-file=/etc/kubernetes/cert/proxy-client-key.pem \\ --service-cluster-ip-range=${SERVICE_CIDR} \\ --service-node-port-range=${NODE_PORT_RANGE} \\ --logtostderr=true \\ --v=2 Restart=on-failure RestartSec=10 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target EOF说明一下：
advertise-address：apiserver 对外通告的 IP（kubernetes 服务后端节点 IP）；default-*-toleration-seconds：设置节点异常相关的阈值；max-*-requests-inflight：请求相关的最大阈值；etcd-*：访问 etcd 的证书和 etcd 服务器地址；experimental-encryption-provider-config：指定用于加密 etcd 中 secret 的配置；bind-address： https 监听的 IP，不能为 127.0.0.1，否则外界不能访问它的安全端口 6443；secret-port：https 监听端口；insecure-port=0：关闭监听 http 非安全端口(8080)；tls-*-file：指定 apiserver 使用的证书、私钥和 CA 文件；audit-*：配置审计策略和审计日志文件相关的参数；client-ca-file：验证 client (kue-controller-manager、kube-scheduler、kubelet、kube-proxy 等)请求所带的证书；enable-bootstrap-token-auth：启用 kubelet bootstrap 的 token 认证；requestheader-*：kube-apiserver 的 aggregator layer 相关的配置参数，proxy-client & HPA 需要使用；requestheader-client-ca-file：用于签名 --proxy-client-cert-file 和 --proxy-client-key-file 指定的证书；在启用了 metric aggregator 时使用；requestheader-allowed-names：不能为空，值为逗号分割的 --proxy-client-cert-file 证书的 CN 名称，这里设置为 “aggregator”；service-account-key-file：签名 ServiceAccount Token 的公钥文件，kube-controller-manager 的 --service-account-private-key-file 定私钥文件，两者配对使用；runtime-config=api/all=true： 启用所有版本的 APIs，如 autoscaling/v2alpha1；authorization-mode=Node,RBAC、–anonymous-auth=false： 开启 Node 和 RBAC 授权模式，拒绝未授权的请求；enable-admission-plugins：启用一些默认关闭的 plugins；allow-privileged：运行执行 privileged 权限的容器；apiserver-count=3：指定 apiserver 实例的数量；event-ttl：指定 events 的保存时间；kubelet-：如果指定，则使用 https 访问 kubelet APIs；需要为证书对应的用户(上面 kubernetes.pem 证书的用户为 kubernetes) 用户定义 RBAC 规则，否则访问 kubelet API 时提示未授权；proxy-client-*：apiserver 访问 metrics-server 使用的证书；service-cluster-ip-range： 指定 Service Cluster IP 地址段；service-node-port-range： 指定 NodePort 的端口范围； 如果 kube-apiserver 机器没有运行 kube-proxy，则还需要添加 --enable-aggregator-routing=true 参数；n、分发kube-apiserver启动文件
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ )) do sed -e "s/##NODE_NAME##/${MASTER_NAMES[i]}/" -e "s/##NODE_IP##/${MASTER_IPS[i]}/" kube-apiserver.service.template > kube-apiserver-${MASTER_IPS[i]}.service done[root@k8s-01 work]# ls kube-apiserver*.service
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-apiserver-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-apiserver.service doneo、启动apiserver
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-apiserver" ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-apiserver && systemctl restart kube-apiserver" done正常图示： p、检查服务是否正常
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "systemctl status kube-apiserver |grep 'Active:'" done正常图示： r、kube-apiserver写入etcd数据
[root@k8s-01 ~]# cd /opt/k8s/work
ETCDCTL_API=3 etcdctl \ --endpoints=${ETCD_ENDPOINTS} \ --cacert=/opt/k8s/work/ca.pem \ --cert=/opt/k8s/work/etcd.pem \ --key=/opt/k8s/work/etcd-key.pem \ get /registry/ --prefix --keys-onlys、检查kube-apiserver监听的端口、检查集群信息 （1）检查kube-apiserver监听的端口
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# netstat -lntup|grep kube 正常图示：
（2）检查集群信息
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# kubectl cluster-info 正常图示： [root@k8s-01 work]# kubectl get all --all-namespaces 正常图示： [root@k8s-01 work]# kubectl get componentstatuses 正常图示：
t、授权kube-apiserver访问kubelet API的权限
[root@k8s-01 ~]# cd /opt/k8s/work
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes正常图示：
该集群包含三个节点，启动后通过竞争选举机制产生一个leader节点，其他节点为阻塞状态。 当leader节点不可用时，阻塞节点将会在此选举产生新的leader，从而保证服务的高可用。
a、创建kube-controller-manager证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > kube-controller-manager-csr.json <<EOF [root@k8s-01 ~]# 添加下面内容：
{ "CN": "system:kube-controller-manager", "key": { "algo": "rsa", "size": 2048 }, "hosts": [ "127.0.0.1", "172.26.16.249", "172.26.16.250", "172.26.16.251" ], "names": [ { "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "system:kube-controller-manager", "OU": "4Paradigm" } ] } EOF说明一下：
host列表包含所有的kube-controller-manager节点IPCN和O均为system:kube-controller-manager，kubernetes 内置的ClusterRoleBindings system:kube-controller-manager赋予kube-controller-manager工作所需权限b、生成证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \ -ca-key=/opt/k8s/work/ca-key.pem \ -config=/opt/k8s/work/ca-config.json \ -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager[root@k8s-01 ~]# ls kube-controller-manager*pem
c、将生成的证书和私钥分发到所有master节点
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-controller-manager*.pem root@${node_ip}:/etc/kubernetes/cert/ doned、创建和分发kubeconfig文件
[root@k8s-01 ~]# cd /opt/k8s/work
（1）创建
kubectl config set-cluster kubernetes \ --certificate-authority=/opt/k8s/work/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube-controller-manager.kubeconfig kubectl config set-credentials system:kube-controller-manager \ --client-certificate=kube-controller-manager.pem \ --client-key=kube-controller-manager-key.pem \ --embed-certs=true \ --kubeconfig=kube-controller-manager.kubeconfig kubectl config set-context system:kube-controller-manager \ --cluster=kubernetes \ --user=system:kube-controller-manager \ --kubeconfig=kube-controller-manager.kubeconfig kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig（2）分发
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-controller-manager.kubeconfig root@${node_ip}:/etc/kubernetes/ donec、创建kube-controller-manager启动文件
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > kube-controller-manager.service.template <<EOF 添加下面内容：
[Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] WorkingDirectory=${K8S_DIR}/kube-controller-manager ExecStart=/opt/k8s/bin/kube-controller-manager \\ --profiling \\ --cluster-name=kubernetes \\ --controllers=*,bootstrapsigner,tokencleaner \\ --kube-api-qps=1000 \\ --kube-api-burst=2000 \\ --leader-elect \\ --use-service-account-credentials\\ --concurrent-service-syncs=2 \\ --bind-address=0.0.0.0 \\ #--secure-port=10252 \\ --tls-cert-file=/etc/kubernetes/cert/kube-controller-manager.pem \\ --tls-private-key-file=/etc/kubernetes/cert/kube-controller-manager-key.pem \\ #--port=0 \\ --authentication-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\ --client-ca-file=/etc/kubernetes/cert/ca.pem \\ --requestheader-allowed-names="" \\ --requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\ --requestheader-extra-headers-prefix="X-Remote-Extra-" \\ --requestheader-group-headers=X-Remote-Group \\ --requestheader-username-headers=X-Remote-User \\ --authorization-kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\ --cluster-signing-cert-file=/etc/kubernetes/cert/ca.pem \\ --cluster-signing-key-file=/etc/kubernetes/cert/ca-key.pem \\ --experimental-cluster-signing-duration=876000h \\ --horizontal-pod-autoscaler-sync-period=10s \\ --concurrent-deployment-syncs=10 \\ --concurrent-gc-syncs=30 \\ --node-cidr-mask-size=24 \\ --service-cluster-ip-range=${SERVICE_CIDR} \\ --pod-eviction-timeout=6m \\ --terminated-pod-gc-threshold=10000 \\ --root-ca-file=/etc/kubernetes/cert/ca.pem \\ --service-account-private-key-file=/etc/kubernetes/cert/ca-key.pem \\ --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \\ --logtostderr=true \\ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target EOF说明一下：
port=0：关闭监听非安全端口（http），同时 –address 参数无效，–bind-address 参数有效；secure-port=10252、–bind-address=0.0.0.0: 在所有网络接口监听 10252 端口的 https /metrics 请求；kubeconfig：指定 kubeconfig 文件路径，kube-controller-manager 使用它连接和验证 kube-apiserver；authentication-kubeconfig 和 –authorization-kubeconfig：kube-controller-manager 使用它连接 apiserver，对 client 的请求进行认证和授权。kube-controller-manager 不再使用 –tls-ca-file 对请求 https metrics 的 Client 证书进行校验。如果没有配置这两个 kubeconfig 参数，则 client 连接 kube-controller-manager https 端口的请求会被拒绝(提示权限不足)。cluster-signing-*-file：签名 TLS Bootstrap 创建的证书；experimental-cluster-signing-duration：指定 TLS Bootstrap 证书的有效期；root-ca-file：放置到容器 ServiceAccount 中的 CA 证书，用来对 kube-apiserver 的证书进行校验；service-account-private-key-file：签名 ServiceAccount 中 Token 的私钥文件，必须和 kube-apiserver 的 –service-account-key-file 指定的公钥文件配对使用；service-cluster-ip-range ：指定 Service Cluster IP 网段，必须和 kube-apiserver 中的同名参数一致；leader-elect=true：集群运行模式，启用选举功能；被选为 leader 的节点负责处理工作，其它节点为阻塞状态；controllers=*,bootstrapsigner,tokencleaner：启用的控制器列表，tokencleaner 用于自动清理过期的 Bootstrap token；horizontal-pod-autoscaler-*：custom metrics 相关参数，支持 autoscaling/v2alpha1；tls-cert-file、–tls-private-key-file：使用 https 输出 metrics 时使用的 Server 证书和秘钥；use-service-account-credentials=true: kube-controller-manager 中各 controller 使用 serviceaccount 访问 kube-apiserver；d、替换启动文件
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ )) do sed -e "s/##NODE_NAME##/${MASTER_NAMES[i]}/" -e "s/##NODE_IP##/${MASTER_IPS[i]}/" kube-controller-manager.service.template > kube-controller-manager-${MASTER_IPS[i]}.service done[root@k8s-01 work]# ls kube-controller-manager*.service
e、分发到所有master节点
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-controller-manager-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-controller-manager.service donef、启动服务
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-controller-manager" ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl restart kube-controller-manager" doneg、检查运行状态、检查服务状态
[root@k8s-01 ~]# cd /opt/k8s/work
(1)检查运行状态
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "systemctl status kube-controller-manager|grep Active" done正常图示： (2)检查运行状态
[root@k8s-01 ~]# netstat -lnpt | grep kube-cont
正常图示：
a、ClusteRole system:kube-controller-manager的权限太小， 只能创建secret、serviceaccount等资源,将controller的权限分散到ClusterRole system:controller:xxx中
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# kubectl describe clusterrole system:kube-controller-manager
正常图示： c、以 deployment controller 为例：
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# kubectl describe clusterrole system:controller:deployment-controller
正常图示： b、 查看当前的 leader
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# kubectl get endpoints kube-controller-manager --namespace=kube-system -o yaml
正常图示：
a、创建 kube-scheduler 证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > kube-scheduler-csr.json <<EOF 添加下面内容：
{ "CN": "system:kube-scheduler", "hosts": [ "127.0.0.1", "172.26.16.249", "172.26.16.250", "172.26.16.251" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "BeiJing", "L": "BeiJing", "O": "system:kube-scheduler", "OU": "4Paradigm" } ] } EOF说明一下：
hosts 列表包含所有 kube-scheduler 节点 IP；CN 和 O 均为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限；b、生成证书和私钥
[root@k8s-01 ~]# cd /opt/k8s/work
cfssl gencert -ca=/opt/k8s/work/ca.pem \ -ca-key=/opt/k8s/work/ca-key.pem \ -config=/opt/k8s/work/ca-config.json \ -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler[root@k8s-01 ~]# ls kube-scheduler*pem
c、将生成的证书和私钥分发到所有 master 节点
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-scheduler*.pem root@${node_ip}:/etc/kubernetes/cert/ doned、创建和分发 kubeconfig 文件 （1）创建
[root@k8s-01 ~]# cd /opt/k8s/work
kubectl config set-cluster kubernetes \ --certificate-authority=/opt/k8s/work/ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=kube-scheduler.kubeconfig kubectl config set-credentials system:kube-scheduler \ --client-certificate=kube-scheduler.pem \ --client-key=kube-scheduler-key.pem \ --embed-certs=true \ --kubeconfig=kube-scheduler.kubeconfig kubectl config set-context system:kube-scheduler \ --cluster=kubernetes \ --user=system:kube-scheduler \ --kubeconfig=kube-scheduler.kubeconfig kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig（2）分发
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-scheduler.kubeconfig root@${node_ip}:/etc/kubernetes/ donee、创建 kube-scheduler 配置文件
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# cat >kube-scheduler.yaml.template <<EOF 添加下面内容：
apiVersion: kubescheduler.config.k8s.io/v1alpha1 kind: KubeSchedulerConfiguration bindTimeoutSeconds: 600 clientConnection: burst: 200 kubeconfig: "/etc/kubernetes/kube-scheduler.kubeconfig" qps: 100 enableContentionProfiling: false enableProfiling: true hardPodAffinitySymmetricWeight: 1 healthzBindAddress: 127.0.0.1:10251 leaderElection: leaderElect: true metricsBindAddress: ##NODE_IP##:10251 EOF说明一下：
kubeconfig：指定 kubeconfig 文件路径，kube-scheduler 使用它连接和验证 kube-apiserver；leader-elect=true：集群运行模式，启用选举功能；被选为 leader 的节点负责处理工作，其它节点为阻塞状态；f、替换模板文件中的变量
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ )) do sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-scheduler.yaml.template > kube-scheduler-${NODE_IPS[i]}.yaml done[root@k8s-01 ~]# ls kube-scheduler*.yaml
g、分发 kube-scheduler 配置文件到所有 master 节点
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-scheduler-${node_ip}.yaml root@${node_ip}:/etc/kubernetes/kube-scheduler.yaml doneh、创建kube-scheduler启动文件
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 ~]# cat > kube-scheduler.service.template <<EOF
[Unit] Description=Kubernetes Scheduler Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] WorkingDirectory=${K8S_DIR}/kube-scheduler ExecStart=/opt/k8s/bin/kube-scheduler \\ --config=/etc/kubernetes/kube-scheduler.yaml \\ --bind-address=##NODE_IP## \\ --secure-port=10259 \\ --port=0 \\ --tls-cert-file=/etc/kubernetes/cert/kube-scheduler.pem \\ --tls-private-key-file=/etc/kubernetes/cert/kube-scheduler-key.pem \\ --authentication-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\ --client-ca-file=/etc/kubernetes/cert/ca.pem \\ --requestheader-allowed-names="" \\ --requestheader-client-ca-file=/etc/kubernetes/cert/ca.pem \\ --requestheader-extra-headers-prefix="X-Remote-Extra-" \\ --requestheader-group-headers=X-Remote-Group \\ --requestheader-username-headers=X-Remote-User \\ --authorization-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \\ --logtostderr=true \\ --v=2 Restart=always RestartSec=5 StartLimitInterval=0 [Install] WantedBy=multi-user.target EOFi、分发配置文件
[root@k8s-01 ~]# cd /opt/k8s/work
for (( i=0; i < 3; i++ )) do sed -e "s/##NODE_NAME##/${NODE_NAMES[i]}/" -e "s/##NODE_IP##/${NODE_IPS[i]}/" kube-scheduler.service.template > kube-scheduler-${NODE_IPS[i]}.service done[root@k8s-01 ~]# ls kube-scheduler*.service
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" scp kube-scheduler-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-scheduler.service donej、启动kube-scheduler
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "mkdir -p ${K8S_DIR}/kube-scheduler" ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-scheduler && systemctl restart kube-scheduler" donek、检查服务运行状态
[root@k8s-01 ~]# cd /opt/k8s/work
for node_ip in ${MASTER_IPS[@]} do echo ">>> ${node_ip}" ssh root@${node_ip} "systemctl status kube-scheduler|grep Active" done正常图示： l、查看输出的 metrics
注意：以下命令在 kube-scheduler 节点上执行kube-scheduler 监听 10251 和 10251 端口：10251：接收 http 请求，非安全端口，不需要认证授权；10259：接收 https 请求，安全端口，需要认证授权；两个接口都对外提供 /metrics 和 /healthz 的访问。[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work]# curl -s http://172.26.16.249:10251/metrics|head
正常图示：
e、查看当前leader
[root@k8s-01 ~]# cd /opt/k8s/work [root@k8s-01 work~]# kubectl get endpoints kube-scheduler --namespace=kube-system -o yaml
正常图示：
