（master 多节点、Nginx/keepalived）
- 环境部署说明
- Kubernetes 多节点部署 （基于单节点）
- 1.拷贝 K8S 工作目录、组件启动脚本至 master02
- 2.master02 修改文件
- 3.搭建 nginx/keepalived
- 4.node 节点修改配置文件
- 5.master 创建 pod 用以测试
- 6.node 节点访问 nginx 页面
环境部署说明
双master节点部署角色如下：
master1 IP地址：192.168.78.55
需求组件：kube-apiserver kube-controller-manager kube-scheduler etcd
master2 IP地址：192.168.78.11
需求组件：kube-apiserver kube-controller-manager kube-scheduler
node1节点 IP地址：192.168.78.66
需求组件：kubelet kube-proxy docker-ce flannel etcd
node2节点 IP地址：192.168.78.77
需求组件：kubelet kube-proxy docker-ce flannel etcd
nginx_1 IP地址：192.168.78.22
需求组件：nginx keepalived
nginx_2 IP地址：192.168.78.33
需求组件：nginx keepalived
VIP IP地址：192.168.78.100Kubernetes 多节点部署 （基于单节点）
1.拷贝 K8S 工作目录、组件启动脚本至 master02
----master01----
'//首先在master01将kubernetes目录传输至master1'
scp -r /opt/kubernetes/ root@192.168.78.11:/opt/
'//然后再将master中三个组件的启动脚本kube-apiserver.service、kube-controller-manager.service、kube-scheduler.service传输至master2'
scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.78.11:/usr/lib/systemd/system
'//master02需要etcd证书'
master1节点操作
拷贝master01上已有的etcd证书给master2使用
PS：因为新加入的master中也包含apiserver，在apiserver工作时，也会需要与ETCD进行交互，所以也需要ETCD证书进行认证
scp -r /opt/etcd/ root@192.168.78.11:/opt/
2.master02 修改文件
-- mster2
hostnamectl set-hostname master2
su -
systemctl stop firewalld && systemctl disable firewalld
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
cd /opt/kubernetes/cfg/
vim kube-apiserver
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--etcd-servers=https://192.168.78.55:2379,https://192.168.78.66:2379,https://192.168.78.66:2379 \ '//这里为etcd集群IP，先不用改'
--bind-address=192.168.78.11 \ '//修改为自身IP'
--secure-port=6443 \
--advertise-address=192.168.78.11 \ '//修改IP'
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--kubelet-https=true \
--enable-bootstrap-token-auth \
--token-auth-file=/opt/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/opt/kubernetes/ssl/server.pem \
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \
--client-ca-file=/opt/kubernetes/ssl/ca.pem \
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/opt/etcd/ssl/ca.pem \
--etcd-certfile=/opt/etcd/ssl/server.pem \
--etcd-keyfile=/opt/etcd/ssl/server-key.pem" '//注意，这里用到了etcd证书'
#apiserver
systemctl start kube-apiserver.service
systemctl enable kube-apiserver.service
systemctl status kube-apiserver.service
#控制器
systemctl start kube-controller-manager.service
systemctl enable kube-controller-manager.service
systemctl status kube-controller-manager.service
#调度器
systemctl start kube-scheduler.service
systemctl enable kube-scheduler.service
systemctl status kube-scheduler.service
ps aux|grep kube
'//检查进程，三个组件是否正常启动'
cat >> /etc/profile << EOF
export PATH=$PATH:/opt/kubernetes/bin/
EOF
source /etc/profile
kubectl get node
'//至此，还没有配置完成，2个node节点只认master01'
'//还需要搭建一个负载均衡群集以完成高可用性，以下使用2个nginx完成'
3.搭建 nginx/keepalived
----nging01、02----
hostnamectl set-hostname nginx01
su -
systemctl stop firewalld && systemctl disable firewalld
setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
vim /etc/yum.repos.d/nginx.repo '//建立nginx的YUM仓库，以便使用yum'
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/ '//指定URL访问路径'
enabled=1 '//开启此yum源，默认项可省略'
gpgcheck=0 '//不验证软件包的签名'
yum install -y nginx
vim /etc/nginx/nginx.conf '//配置nginx，添加四层转发'
9 events {
10 worker_connections 1024;
11 } '//插入以下内容'
12 stream {
13
14 log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent'; '//日志格式'
15 access_log /var/log/nginx/k8s-access.log main; '//K8S日志存放路径'
16
17 upstream k8s-apiserver { '//配置负载均衡，指向master'
18 server 192.168.78.55:6443;
19 server 192.168.78.11:6443;
20 }
21 server {
22 listen 6443; '//访问端口'
23 proxy_pass k8s-apiserver; '//转发调动proxy访问代理'
24 }
25 }
26
nginx -t '//检查语法'
systemctl start nginx && systemctl enable nginx
netstat -natp|grep nginx
*************************************************************
----nging01、02----
'//部署keepalived服务'
yum -y install keepalived
vim /etc/keepalived/keepalived.conf
'//修改配置文件'
! Configuration File for keepalived
global_defs { '//收邮件地址 '
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
} '//邮件发送地址 '
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
vrrp_script check_nginx { '//检查nginx的服务脚本，与之相关联'
script "/etc/nginx/check_nginx.sh"
}
vrrp_instance VI_1 {
state MASTER '//注：nginx02为BACKUP'
interface ens33
virtual_router_id 51 'VRRP路由ID实例，每个实例是唯一的'
priority 100 '优先级，nginx02设置90'
advert_int 1 '指定VRRP心跳包通告间隔时间，默认1秒'
authentication {
auth_type PASS
auth_pass 123123
}
virtual_ipaddress {
192.168.78.100/24 '//VIP'
}
track_script {
check_nginx '//检测nginx，触发keepalived'
}
}
**************************************************************
----nging01、02----
vim /etc/nginx/check_nginx.sh
'//count为变量，用于统计'
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")
'//过滤nginx进程'
if [ "$count" -eq 0 ];then '//0表示nginx没有运行'
systemctl stop keepalived '//则关闭keepalived'
fi
chmod +x /etc/nginx/check_nginx.sh '//赋权'
----nginx01、02---
systemctl start keepalived.service && systemctl enable keepalived.service
ip a '//显示网络设备，查看VIP'
inet 192.168.126.100/24 scope global secondary ens33
----nginx02----
systemctl start keepalived.service && systemctl enable keepalived.service
systemctl status keepalived.service
ip a
'VIP在nginx01中'
'//可在nginx01中关闭nginx服务，再去nginx02使用ip a查看VIP是否漂移'
'//恢复操作：回到nginx01中重新启动nginx与keepalived，VIP就会漂移回来，nginx01优先级比nginx02高'
4.node 节点修改配置文件
----node01、02---
'//开始修改node节点中配置文件，统一为VIP'
vim /opt/kubernetes/cfg/bootstrap.kubeconfig
vim /opt/kubernetes/cfg/kubelet.kubeconfig
vim /opt/kubernetes/cfg/kube-proxy.kubeconfig
server: https://192.168.78.100:6443
systemctl restart kubelet.service kube-proxy.service
cd /opt/kubernetes/cfg/
grep 100 * '//过滤检查'
----nginx01----
systemctl restart nginx
tail /var/log/nginx/k8s-access.log '//查看K8S日志，确认实现负载均衡'
5.master 创建 pod 用以测试
----master01----
'//创建pod用以测试'
kubectl run nginx --image=nginx
kubectl get pods '//等待一会儿查看，处于创建状态，Running为成功'
'//查看pod日志'
kubectl logs nginx-dbddb74b8-rwz94
Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-rwz94)
'//报错，不具备对应权限查看日志，以下为解决办法'
kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous
kubectl logs nginx-dbddb74b8-rwz94
kubectl describe pod nginx-dbddb74b8-rwz94
'//可进一步查看详细信息'
'//还可以去对应node节点查看docker，已经创建相应容器（基础）'
kubectl get pods -o wide '//查看所创建pods的IP'
'//192.168.78.66为node01，下一步去node01节点查看'
6.node 节点访问 nginx 页面
----node01----
'//在对应网段的node节点上可以访问nginx页面'
[root@node01 cfg]# curl 172.17.16.3
----master01----
'//node01访问nginx页面后，会产生日志，回到master01查看'
kubectl logs nginx-dbddb74b8-rwz94
