Это 30-й день моего участия в Gengwen Challenge. Чтобы узнать подробности о мероприятии, пожалуйста, проверьте:Обновить вызов
Один kubeadm должен изменить исходный код
Кластер k8s, установленный kubeadm, имеет проблему с сертификатом.Сертификат действителен в течение одного года.Если срок его действия истечет, команда kubectl будет ненормальной. Его можно редактировать и компилировать.
1.1 Загрузите исходный код
# 下载源码
[root@master common-service]# git clone https://github.com.cnpmjs.org/kubernetes/kubernetes.git
[root@master common-service]# cd kubernetes
[root@master kubernetes]# kubectl version
Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.5", GitCommit:"20c265fef0741dd71a66480e35bd69f18351daea", GitTreeState:"clean", BuildDate:"2019-10-15T19:07:57Z", GoVersion:"go1.12.10", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.5", GitCommit:"20c265fef0741dd71a66480e35bd69f18351daea", GitTreeState:"clean", BuildDate:"2019-10-15T19:07:57Z", GoVersion:"go1.12.10", Compiler:"gc", Platform:"linux/amd64"}
[root@master kubernetes]# git checkout release-1.15
Branch release-1.15 set up to track remote branch release-1.15 from origin.
Switched to a new branch 'release-1.15'
# git checkout release-1.16
[root@master kubernetes]# git branch
master
* release-1.15
[root@master kubernetes]# vim cmd/kubeadm/app/util/pkiutil/pki_helpers.go
[root@master kubernetes]# git diff
diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
index e1d77e1..755c08a 100644
--- a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
+++ b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go
@@ -571,7 +571,7 @@ func NewSignedCert(cfg *certutil.Config, key crypto.Signer, caCert *x509.Certifi
IPAddresses: cfg.AltNames.IPs,
SerialNumber: serial,
NotBefore: caCert.NotBefore,
- NotAfter: time.Now().Add(kubeadmconstants.CertificateValidity).UTC(),
+ NotAfter: time.Now().Add(kubeadmconstants.CertificateValidity * 100).UTC(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: cfg.Usages,
}
# 编译源码，需要在服务器上运行有go环境
1.2 Установка рабочей среды
cd /opt
wget -c https://dl.google.com/go/go1.12.7.linux-amd64.tar.gz
tar -C /usr/local -zxvf go1.12.7.linux-amd64.tar.gz
cat >>/etc/profile <<EOF
export GOROOT=/usr/local/go
export PATH=\$PATH:\$GOROOT/bin
EOF
source /etc/profile
cat >> /root/.bash_profile <<EOF
export GOPATH=\$HOME/go
EOF
source /root/.bash_profile
1.3 Компиляция kubeadm
[root@VM-16-3-centos kubernetes_bak]# KUBE_BUILD_PLATFORMS=linux/amd64 make all WHAT=cmd/kubeadm GOFLAGS=-v GOGCFLAGS="-N -l"
# 编译完成后在_output/bin目录下
[root@VM-16-3-centos kubernetes]# ls _output/bin/
conversion-gen deepcopy-gen defaulter-gen go2make go-bindata kubeadm openapi-gen
[root@VM-16-3-centos kubernetes]# cd _output/bin/
[root@VM-16-3-centos bin]# ./kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"15+", GitVersion:"v1.15.13-beta.0.1+a34f1e483104bd", GitCommit:"a34f1e483104bd51c3e9a6aec3dbbcf6301789da", GitTreeState:"clean", BuildDate:"2020-08-25T10:45:53Z", GoVersion:"go1.12.7", Compiler:"gc", Platform:"linux/amd64"}
1.4 Обновите сертификат
# 将生成好的kubeadm文件上传到k8s集群的各master上
[root@master kubernetes]# /root/kubeadm alpha certs check-expiration
failed to load existing certificate apiserver-etcd-client: open /etc/kubernetes/pki/apiserver-etcd-client.crt: no such file or directory
[root@master kubernetes]# /root/kubeadm alpha certs renew all
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
failed to load existing certificate apiserver-etcd-client: open /etc/kubernetes/pki/apiserver-etcd-client.crt: no such file or directory
# 发现利用kubesphere安装的etcd证书异常，那么，手动更新几个证书
# 1.更新api-server证书
[root@master kubernetes]# /root/kubeadm alpha certs renew apiserver
certificate for serving the Kubernetes API renewed
[root@master kubernetes]# openssl x509 -in ssl/apiserver.crt -noout -enddate
notAfter=Aug 1 15:35:33 2120 GMT
# 2.更新apiserver-kubelet-client 证书
[root@master kubernetes]# /root/kubeadm alpha certs renew apiserver-kubelet-client
certificate for the API server to connect to kubelet renewed
[root@master kubernetes]# openssl x509 -in ssl/apiserver-kubelet-client.crt -noout -text |grep Not
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 1 15:30:40 2120 GMT
# 3.更新front-proxy-client证书
[root@master kubernetes]# /root/kubeadm alpha certs renew front-proxy-client
certificate for the front proxy client renewed
[root@master kubernetes]# openssl x509 -in ssl/front-proxy-client.crt -noout -text |grep Not
Not Before: Sep 3 07:13:08 2019 GMT
Not After : Aug 1 15:36:35 2120 GMT
Массовые операции
[root@master ~]# for i in `ls /etc/kubernetes/pki/*.crt`;do echo $i;openssl x509 -in ${i} -noout -text |grep "Not";done
/etc/kubernetes/pki/apiserver.crt
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 31 10:55:38 2021 GMT
/etc/kubernetes/pki/apiserver-kubelet-client.crt
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 31 10:55:39 2021 GMT
/etc/kubernetes/pki/ca.crt
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 31 07:13:10 2029 GMT
/etc/kubernetes/pki/front-proxy-ca.crt
Not Before: Sep 3 07:13:08 2019 GMT
Not After : Aug 31 07:13:08 2029 GMT
/etc/kubernetes/pki/front-proxy-client.crt
Not Before: Sep 3 07:13:08 2019 GMT
Not After : Aug 31 10:55:39 2021 GMT
[root@xuel-ksserver data]# KUBECMD=/data/kubeadm16
[root@xuel-ksserver data]# for i in apiserver apiserver-kubelet-client front-proxy-client;do ${KUBECMD} alpha certs renew ${i};done
certificate for serving the Kubernetes API renewed
certificate for the API server to connect to kubelet renewed
certificate for the front proxy client renewed
[root@master ~]# for i in `ls /etc/kubernetes/pki/*.crt`;do echo $i;openssl x509 -in ${i} -noout -text |grep "Not";done
/etc/kubernetes/pki/apiserver.crt
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 7 11:06:09 2120 GMT
/etc/kubernetes/pki/apiserver-kubelet-client.crt
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 7 11:06:10 2120 GMT
/etc/kubernetes/pki/ca.crt
Not Before: Sep 3 07:13:10 2019 GMT
Not After : Aug 31 07:13:10 2029 GMT
/etc/kubernetes/pki/front-proxy-ca.crt
Not Before: Sep 3 07:13:08 2019 GMT
Not After : Aug 31 07:13:08 2029 GMT
/etc/kubernetes/pki/front-proxy-client.crt
Not Before: Sep 3 07:13:08 2019 GMT
Not After : Aug 7 11:06:10 2120 GMT
# 检查etcd证书
[root@master ~]# for i in `ls /etc/kubernetes/pki/etcd/*.pem`;do echo $i;openssl x509 -in ${i} -noout -dates ;done
/etc/kubernetes/pki/etcd/admin-master-key.pem
unable to load certificate
140633169307552:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/admin-master.pem
notBefore=Sep 3 07:11:37 2019 GMT
notAfter=Aug 10 07:11:37 2119 GMT
/etc/kubernetes/pki/etcd/ca-key.pem
unable to load certificate
140597254109088:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/ca.pem
notBefore=Sep 3 07:11:36 2019 GMT
notAfter=Aug 10 07:11:36 2119 GMT
/etc/kubernetes/pki/etcd/member-master-key.pem
unable to load certificate
140164986681248:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/member-master.pem
notBefore=Sep 3 07:11:36 2019 GMT
notAfter=Aug 10 07:11:36 2119 GMT
/etc/kubernetes/pki/etcd/node-master-key.pem
unable to load certificate
139708689217440:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/node-master.pem
notBefore=Sep 3 07:11:37 2019 GMT
notAfter=Aug 10 07:11:37 2119 GMT
/etc/kubernetes/pki/etcd/node-node01-key.pem
unable to load certificate
140027733841824:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/node-node01.pem
notBefore=Sep 3 07:11:38 2019 GMT
notAfter=Aug 10 07:11:38 2119 GMT
/etc/kubernetes/pki/etcd/node-node02-key.pem
unable to load certificate
140489035491232:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/kubernetes/pki/etcd/node-node02.pem
notBefore=Sep 3 07:11:38 2019 GMT
notAfter=Aug 10 07:11:38 2119 GMT
Два обновления всех
# 备份相关配置文件和ssl证书
mkdir /etc/kubernetes.bak
cp -r /etc/kubernetes/pki/ /etc/kubernetes.bak
cp /etc/kubernetes/*.conf /etc/kubernetes.bak
cp -r /var/lib/etcd /var/lib/etcd.bak
# 上传已经修改好的kubeadm文件
# 全部替换证书，在所有master节点均操作
[root@master ~]#
[root@master ~]# for file in `./kubeadm alpha certs check-expiration --config=/etc/kubernetes/kubeadm-config.yaml | awk 'NR>1{print $1}'`;do ./kubeadm alpha certs renew $file;done
certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed
# 检测证书
[root@master ~]# ./kubeadm alpha certs check-expiration --config=/etc/kubernetes/kubeadm-config.yaml
CERTIFICATE EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
admin.conf Aug 13, 2120 02:52 UTC 99y no
apiserver Aug 13, 2120 02:52 UTC 99y no
apiserver-kubelet-client Aug 13, 2120 02:52 UTC 99y no
controller-manager.conf Aug 13, 2120 02:52 UTC 99y no
front-proxy-client Aug 13, 2120 02:52 UTC 99y no
scheduler.conf Aug 13, 2120 02:52 UTC 99y no
[root@master ~]# for i in `ls /etc/kubernetes/pki/*.crt`;do echo $i;openssl x509 -in ${i} -noout -text |grep "Not";done
/etc/kubernetes/pki/apiserver.crt
Not Before: Sep 5 03:13:03 2020 GMT
Not After : Aug 13 02:52:53 2120 GMT
/etc/kubernetes/pki/apiserver-kubelet-client.crt
Not Before: Sep 5 03:13:03 2020 GMT
Not After : Aug 13 02:52:54 2120 GMT
/etc/kubernetes/pki/ca.crt
Not Before: Sep 5 03:13:03 2020 GMT
Not After : Sep 3 03:13:03 2030 GMT
/etc/kubernetes/pki/front-proxy-ca.crt
Not Before: Sep 5 03:13:04 2020 GMT
Not After : Sep 3 03:13:04 2030 GMT
/etc/kubernetes/pki/front-proxy-client.crt
Not Before: Sep 5 03:13:04 2020 GMT
Not After : Aug 13 02:52:56 2120 GMT
# 查看etcd证书
[root@master ~]# for i in `ls /etc/ssl/etcd/ssl/*.pem`;do echo $i;openssl x509 -in ${i} -noout -dates ;done
/etc/ssl/etcd/ssl/admin-master-key.pem
unable to load certificate
139850633099168:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/admin-master.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/ca-key.pem
unable to load certificate
140344601663392:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/ca.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/member-master-key.pem
unable to load certificate
140576245700512:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/member-master.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-master-key.pem
unable to load certificate
140665549584288:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-master.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-node01-key.pem
unable to load certificate
140005720024992:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-node01.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-node02-key.pem
unable to load certificate
139824797161376:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-node02.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
/etc/ssl/etcd/ssl/node-node03-key.pem
unable to load certificate
139721134917536:error:0906D06C:PEM routines:PEM_read_bio:no start line:pem_lib.c:707:Expecting: TRUSTED CERTIFICATE
/etc/ssl/etcd/ssl/node-node03.pem
notBefore=Sep 5 03:10:39 2020 GMT
notAfter=Aug 12 03:10:39 2120 GMT
# 重启控制层面
docker ps | grep -v pause | grep -E "etcd|scheduler|controller|apiserver" | awk '{print $1}' | awk '{print "docker","restart",$1}' | bash
#重启
systemctl restart kubelet
# 恢复
cp config config.bak
cp /etc/kubernetes/admin.conf config
Уведомление
- Компиляция kubernetes 1.15 соответствует использованию go1.12.7, с более высокими версиями будут проблемы
- Компиляция kubernetes 1.16 соответствует использованию go1.13.4, с более высокими версиями будут проблемы
- Поскольку сертификату ca уже 10 лет, его все равно необходимо обновить через десять лет.
[root@VM-16-3-centos kubernetes_bak]# git branch
master
* release-1.16
[root@VM-16-3-centos kubernetes_bak]# go version
go version go1.13.6 linux/amd64
- После повторного создания сертификата необходимо перезапустить плоскость управления, чтобы он вступил в силу.