make backup
sudo mkdir backup
sudo cp -R /etc/kubernetes backup/
sudo tar -cvzf backup/pki_backup_`hostname`-`date +%Y%m%d`.tar.gz backup/kubernetes/
look to dir /etc/kubernetes/
ls -l
total 80
-rw------- 1 root root 5440 Mar 3 13:21 admin.conf
drwxr-xr-x 2 root root 4096 Aug 17 2020 audit-policy
-rw-r--r-- 1 root root 368 Mar 4 2020 calico-config.yml
-rw-r--r-- 1 root root 270 Mar 4 2020 calico-crb.yml
-rw-r--r-- 1 root root 341 Mar 4 2020 calico-cr.yml
-rw-r--r-- 1 root root 147 Mar 4 2020 calico-node-sa.yml
-rw-r--r-- 1 root root 6363 Mar 4 2020 calico-node.yml
-rw------- 1 root root 5472 Mar 3 13:21 controller-manager.conf
-rw-r--r-- 1 root root 3041 Aug 14 2020 kubeadm-config.v1alpha3.yaml
-rw------- 1 root root 5548 Mar 3 13:21 kubelet.conf
-rw-r--r-- 1 root root 1751 Mar 4 2020 kubelet.env
drwxr-xr-x 2 kube root 4096 Aug 14 2020 manifests
lrwxrwxrwx 1 root root 28 Mar 4 2020 node-kubeconfig.yaml -> /etc/kubernetes/kubelet.conf
-rw------- 1 root root 5420 Mar 3 13:21 scheduler.conf
drwxr-xr-x 3 kube root 4096 Mar 3 10:20 ssl
try to find k8s cluster config, in my case this is
kubeadm-config.v1alpha3.yaml
if same file absent you can generate
kubectl get cm kubeadm-config -n kube-system -o yaml > /etc/kubernetes/kubeadm-config.yaml
also in my case absent folder /etc/kubernetes/pki and exist ~/ssl
I create symlink /etc/kubernetes/pki from /etc/kubernetes/ssl
ln -s /etc/kubernetes/ssl /etc/kubernetes/pki
let regenerate certs
kubeadm alpha phase certs apiserver --config /etc/kubernetes/kubeadm-config.v1alpha3.yaml
[certificates] Using the existing apiserver certificate and key.
kubeadm alpha phase certs apiserver-kubelet-client
I0303 13:12:24.543254 40613 version.go:236] remote version is much newer: v1.20.4; falling back to: stable-1.12
[certificates] Using the existing apiserver-kubelet-client certificate and key.
kubeadm alpha phase certs front-proxy-client
I0303 13:12:35.660672 40989 version.go:236] remote version is much newer: v1.20.4; falling back to: stable-1.12
[certificates] Using the existing front-proxy-client certificate and key.
kubeadm alpha phase certs etcd-server --config /etc/kubernetes/kubeadm-config.v1alpha3.yaml
[certificates] Generated etcd/server certificate and key.
[certificates] etcd/server serving cert is signed for DNS names [prod-uct1-mvp-k8s-0001 localhost] and IPs [127.0.0.1 ::1]
kubeadm alpha phase certs etcd-server --config /etc/kubernetes/kubeadm-config.v1alpha3.yaml
[certificates] Using the existing etcd/server certificate and key.
kubeadm alpha phase certs etcd-healthcheck-client --config /etc/kubernetes/kubeadm-config.v1alpha3.yaml
[certificates] Generated etcd/healthcheck-client certificate and key.
kubeadm alpha phase certs etcd-peer --config /etc/kubernetes/kubeadm-config.v1alpha3.yaml
[certificates] Generated etcd/peer certificate and key.
[certificates] etcd/peer serving cert is signed for DNS names [product1-mvp-k8s-0001 localhost] and IPs [192.168.4.201 127.0.0.1 ::1]
check actuality certs
find /etc/kubernetes/pki/ -name '*.crt' -exec openssl x509 -text -noout -in {} \; | grep -A2 Validity
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 2 10:29:44 2030 GMT
--
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 3 10:07:29 2022 GMT
--
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 3 10:07:52 2022 GMT
--
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 3 10:06:48 2022 GMT
--
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 2 10:29:44 2030 GMT
--
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 2 19:39:56 2022 GMT
--
Validity
Not Before: Mar 4 10:29:43 2020 GMT
Not After : Mar 2 10:29:43 2030 GMT
--
Validity
Not Before: Mar 4 10:29:43 2020 GMT
Not After : Mar 2 19:40:13 2022 GMT
--
Validity
Not Before: Mar 4 10:29:44 2020 GMT
Not After : Mar 2 19:36:38 2022 GMT
next step to generate new config files: admin.conf, controller-manager.conf, kubelet.conf, scheduler.conf, in first step move old files to /tmp
cd /etc/kubernetes/
mv {admin.conf,controller-manager.conf,kubelet.conf,scheduler.conf} /tmp/
kubeadm alpha phase kubeconfig all --config /etc/kubernetes/kubeadm-config.v1alpha3.yaml
[kubeconfig] Using existing up-to-date KubeConfig file: "/etc/kubernetes/admin.conf"
[kubeconfig] Using existing up-to-date KubeConfig file: "/etc/kubernetes/kubelet.conf"
[kubeconfig] Using existing up-to-date KubeConfig file: "/etc/kubernetes/controller-manager.conf"
[kubeconfig] Using existing up-to-date KubeConfig file: "/etc/kubernetes/scheduler.conf"
after restart master kubelet and check status after restart
sudo systemctl stop kubelet; sudo docker stop $(docker ps -aq); sudo docker rm $(docker ps -aq); sudo systemctl start kubelet
systemctl status kubelet -l
● kubelet.service - Kubernetes Kubelet Server
Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2021-03-03 14:00:22 MSK; 10s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Process: 52998 ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volume-plugins (code=exited, status=0/SUCCESS)
Main PID: 53001 (kubelet)
Memory: 51.2M
CGroup: /system.slice/kubelet.service
check access master node
kubectl get nodes
kubectl get ns
NAME STATUS AGE
default Active 464d
product1-mvp Active 318d
infra-logging Active 315d
infra-nginx-ingress Active 386d
kube-public Active 464d
kube-system Active 464d
pg Active 318d
check cert
notAfter=Mar 3 07:40:43 2022 GMT
repeat procedure for all master nodes.
Now we ready to renew certs for work nodes.
In first step we must delete or move kubelet.conf
cd /etc/kubernetes/
mv kubelet.conf kubelet.conf_old
after change bootstrap-kubelet.conf
**apiVersion: v1
clusters:
- cluster:
certificate-authority-data: | LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01ETX
server: https://192.168.4.201:6443
name: product1
contexts:
- context:
cluster: product1
user: tls-bootstrap-token-user
name: tls-bootstrap-token-user@product1
current-context: tls-bootstrap-token-user@product1
kind: Config
preferences: {}
users:
- name: tls-bootstrap-token-user
user:
token: fgz9qz.lujw0bwsdfhdsfjhgds**
where:
certificate-authority-data – root certificate PKI CA master, may take from /etc/kubernetes/kubelet.conf on master nodes
token: fgz9qz.lujw0bwsdfhdsfjhgds - token, generate on master node
kubeadm token create
after restart and check kubelet work node and work node come back to cluster:
systemctl restart kubelet
systemctl status kubelet -l
● kubelet.service - Kubernetes Kubelet Server
Loaded: loaded (/etc/systemd/system/kubelet.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2021-03-03 14:06:33 MSK; 11s ago
Docs: https://github.com/GoogleCloudPlatform/kubernetes
Process: 54615 ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volume-plugins (code=exited, status=0/SUCCESS)
Main PID: 54621 (kubelet)
Memory: 52.1M
CGroup: /system.slice/kubelet.service
check update certs may be ls in dir
ls -las /var/lib/kubelet/pki/
total 24
4 -rw-------. 1 root root 1135 Mar 3 14:06 kubelet-client-2021-03-03-14-06-34.pem
0 lrwxrwxrwx. 1 root root 59 Mar 3 14:06 kubelet-client-current.pem -> /var/lib/kubelet/pki/kubelet-client-2021-03-03-14-06-34.pem
4 -rw-r--r--. 1 root root 2267 Mar 2 10:40 kubelet.crt
4 -rw-------. 1 root root 1679 Mar 2 10:40 kubelet.key
after repeat this procedure by all work nodes.