1、环境准备
IP | 主机名 | 角色 | OS |
10.0.0.17 | wang-17.host.com | 负载均衡、内网DNS | Centos7.9 |
10.0.0.27 | wang-27.host.com | 负载均衡 | Centos7.9 |
10.0.0.37 | wang-37.host.com | k8s master、k8s node、etcd | Centos7.9 |
10.0.0.47 | wang-47.host.com | k8s master、k8s node、etcd | Centos7.9 |
10.0.0.200 | wang-200.host.com | harbor私有仓库、发布节点 | Centos7.9 |
10.0.0.99 | VIP |
组件 | 版本 | 备注 |
harbor | 1.10.15 | 私有仓库 |
etcd | 3.1.20 | 数据库 |
k8s | 1.15.4 | k8s主要组件 |
温馨提示： 所有节点执行
[root@wang-17 ~]# systemctl stop firewalld && systemctl disable firewalld && systemctl status firewalld
[root@wang-17 ~]# sed -i 's/enforcing/disabled/' /etc/selinux/config
[root@wang-17 ~]# setenforce 0
[root@wang-17 ~]# yum install -y epel-release
[root@wang-17 ~]# yum install -y chrony
[root@wang-17 ~]# systemctl start chronyd
[root@wang-17 ~]# systemctl status chronyd
[root@wang-17 ~]# chronyc sources
2、部署dns
温馨提示：wang-17节点执行
[root@wang-17 ~]# yum install bind bind-utils -y
[root@wang-17 ~]# vim /etc/named.conf
[root@wang-17 ~]# cat /etc/named.conf
......
listen-on port 53 { 10.0.0.17; }; #更改
// listen-on-v6 port 53 { ::1; }; #注释
allow-query { any; }; #更改
forwarders { 10.0.0.2; }; #更改
recursion yes;
dnssec-enable no; #更改
dnssec-validation no; #更改
......
#增加主机域host.com 和业务员wang.com
[root@wang-17 ~]# vim /etc/named.rfc1912.zones
......
zone "host.com" IN {
type master;
file "host.com.zone";
allow-update { 10.0.0.17; };
};
zone "wang.com" IN {
type master;
file "wang.com.zone";
allow-update { 10.0.0.17; };
};
[root@wang-17 ~]# cd /var/named/
[root@wang-17 named]# cp -p named.localhost host.com.zone
[root@wang-17 named]# vim host.com.zone
$TTL 1D
@ IN SOA dns.host.com. dnsadmin.host.com. (
2022112501 ; serial
108000 ; refresh # (3 hours)
900 ; retry # (15 minutes)
604800 ; expire # (1 week)
86400 ) ; minimum # (1 day)
NS dns.host.com.
dns A 10.0.0.17
wang-17 A 10.0.0.17
wang-27 A 10.0.0.27
wang-37 A 10.0.0.37
wang-47 A 10.0.0.47
wang-200 A 10.0.0.200
[root@wang-17 named]# cp -p host.com.zone wang.com.zone
[root@wang-17 named]# vim wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112501 ; serial
108000 ; refresh # (3 hours)
900 ; retry # (15 minutes)
604800 ; expire # (1 week)
86400 ) ; minimum # (1 day)
NS dns.wang.com.
dns A 10.0.0.17
[root@wang-17 named]# named-checkconf
[root@wang-17 named]# systemctl start named && systemctl enable named && systemctl status named
[root@wang-17 named]# dig -t A wang-200.host.com @10.0.0.17 +short
10.0.0.200
#温馨提示： 所有节点执行
[root@wang-17 named]# sed -i 's/^DNS1=.*/DNS1=10.0.0.17/g' /etc/sysconfig/network-scripts/ifcfg-eth0
[root@wang-17 named]# systemctl restart network
#温馨提示： windows解析域名建议使用switchhosts软件
3、根证书准备
问下提示：wang-200节点执行
[root@wang-200 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.3/cfssl_1.6.3_linux_amd64 -O /usr/local/bin/cfssl
[root@wang-200 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.3/cfssljson_1.6.3_linux_amd64 -O /usr/local/bin/cfssl-json
[root@wang-200 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.3/cfssl-certinfo_1.6.3_linux_amd64 -O /usr/local/bin/cfssl-certinfo
[root@wang-200 ~]# chmod u+x /usr/local/bin/cfssl*
[root@wang-200 ~]# ll /usr/local/bin/cfssl*
-rwxr-xr-x 1 root root 13336752 Oct 5 01:43 /usr/local/bin/cfssl
-rwxr-xr-x 1 root root 10795824 Oct 5 01:41 /usr/local/bin/cfssl-certinfo
-rwxr-xr-x 1 root root 7646320 Oct 5 01:42 /usr/local/bin/cfssl-json
#温馨提示：
CN: Common Name，浏览器使用该字段验证网站是否合法，一般写的是域名。非常重要。浏览器使用该字段验证网站是否合法
C: Country，国家
ST: State，州，省
L: Locality，地区，城市
O: Organization Name，组织名称，公司名称
OU: Organization Unit Name，组织单位名称，公司部门
[root@wang-200 certs]# tee ca-csr.json <<'EOF'
{
"CN": "WANG",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "Ali",
"OU": "ops"
}
],
"ca": {
"expiry": "175200h"
}
}
EOF
[root@wang-200 certs]# cfssl gencert -initca ca-csr.json | cfssl-json -bare ca
[root@wang-200 certs]# ls
ca.csr ca-csr.json ca-key.pem ca.pem
4、docker环境准备
温馨提示：wang-37、wang-47、wang-200节点执行
[root@wang-200 certs]# wget -O /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
或者使用一条命令在线安装：curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun
#温馨提示：
# 不安全的registry中增加了harbor地址
# 各个机器上bip网段==不一致==，bip中间两段与宿主机最后两段相同，目的是方便定位问题
# 如果复制内容有问题，建议手动输入，有可能复制会有不可见字符导致docker无法启动
[root@wang-200 certs]# mkdir /etc/docker -p
[root@wang-200 certs]# vim /etc/docker/daemon.json
{
"graph": "/data/docker",
"storage-driver": "overlay2",
"insecure-registries": ["harbor.wang.com","quay.io"],
"bip": "172.7.200.1/24",
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://pgavrk5n.mirror.aliyuncs.com"],
"live-restore": true
}
[root@wang-200 certs]# systemctl daemon-reload
[root@wang-200 certs]# systemctl start docker
[root@wang-200 certs]# systemctl enable docker
5、部署harbor
温馨提示：wang-200 节点执行
# 说明：
# /opt/src : 源码、文件下载目录
# /opt/release : 各个版本软件存放位置
# /opt/apps : 各个软件当前版本的软链接
[root@wang-200 certs]# mkdir /opt/src
[root@wang-200 certs]# mkdir /opt/release
[root@wang-200 certs]# mkdir /opt/apps
[root@wang-200 certs]# cd /opt/src
[root@wang-200 src]# wget https://github.com/goharbor/harbor/releases/download/v1.10.15/harbor-offline-installer-v1.10.15.tgz
[root@wang-200 src]# tar xf harbor-offline-installer-v1.10.15.tgz
[root@wang-200 src]# mv harbor /opt/release/harbor-v1.10.15
[root@wang-200 src]# ln -s /opt/release/harbor-v1.10.15 /opt/apps/harbor
[root@wang-200 src]# ll /opt/apps/harbor
lrwxrwxrwx 1 root root 28 Nov 25 19:51 /opt/apps/harbor -> /opt/release/harbor-v1.10.15
[root@wang-200 src]# cd /opt/apps/harbor
[root@wang-200 harbor]# ls
common.sh harbor.v1.10.15.tar.gz harbor.yml install.sh LICENSE prepare
[root@wang-200 harbor]# vim harbor.yml
hostname: harbor.wang.com
http:
port: 180
harbor_admin_password: harbor@12345
data_volume: /data/harbor
location: /data/harbor/logs
[root@wang-200 harbor]# yum install -y docker-compose
[root@wang-200 harbor]# ./install.sh
#开机自启方法一，使用systemd管理harbor
[root@wang-200 harbor]# docker-compose down
[root@wang-200 harbor]# vim /usr/lib/systemd/system/harbor.service
[Unit]
Description=Harbor
After=docker.service systemd-networkd.service systemd-resolved.service
Requires=docker.servcie
Documentation=http://github.com/vmware/harbor
[Service]
Type=simple
Restart=on-faliure
RestartSec=5
ExecStart=/usr/bin/docker-compose -f /opt/apps/harbor/docker-compose.yml up
ExecStop=/usr/bin/docker-compose -f /opt/apps/harbor/docker-compose.yml down
[Install]
WantedBy=multi-user.target
[root@wang-200 harbor]# systemctl daemon-reload
[root@wang-200 harbor]# systemctl start harbor
[root@wang-200 harbor]# systemctl status harbor
[root@wang-200 harbor]# systemctl enable harbor
#开机自启动方法二：
cat >> /etc/rc.d/rc.local <<EOF
# start harbor
cd /opt/apps/harbor
/usr/bin/docker-compose stop
/usr/bin/docker-compose start
EOF
# 注意要给执行权限
chmod +x /etc/rc.d/rc.local
6、部署nginx反向代理harbor
温馨提示：wang-200节点执行：
# 说明：
# 当前机器中Nginx功能较少，使用yum安装即可。如有多个harbor考虑源码编译且配置健康检查
# nginx配置此处忽略，仅仅使用最简单的配置。
[root@wang-200 ~]# yum install -y nginx
[root@wang-200 ~]# vim /etc/nginx/conf.d/harbor.conf
server {
listen 80;
server_name harbor.wang.com;
client_max_body_size 1g; # 避免出现上传失败的情况，此处设置上次大小为1G
location / {
proxy_pass http://127.0.0.1:180;
}
}
[root@wang-200 ~]# nginx -t
[root@wang-200 ~]# systemctl start nginx
[root@wang-200 ~]# systemctl status nginx
[root@wang-200 ~]# systemctl enable nginx
# 温馨提示：wang-17节点执行
# 解析harbor.wang.com域名
[root@wang-17 named]# vim wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112502 ; serial
108000 ; refresh
900 ; retry
604800 ; expire
86400 ) ; minimum
NS dns.wang.com.
dns A 10.0.0.17
harbor A 10.0.0.200
[root@wang-17 named]# systemctl restart named
[root@wang-17 named]# systemctl status named
[root@wang-17 named]# dig -t A harbor.wang.com @10.0.0.17 +short
10.0.0.200
[root@wang-17 named]# host harbor.wang.com
harbor.wang.com has address 10.0.0.200
# 测试harbor
[root@wang-200 ~]# docker pull nginx:1.22
[root@wang-200 ~]# docker tag nginx:1.22 harbor.wang.com/public/nginx:1.22
[root@wang-200 ~]# docker login harbor.wang.com
Username: admin
Password:
[root@wang-200 ~]# docker push harbor.wang.com/public/nginx:1.22
7、部署etcd
说明： etcd 的leader选举机制，要求至少为3台或以上的奇数
7-1、签发etcd证书
温馨提示：wang-200节点执行
说明：证书签发服务器：wang-200
创建ca的json配置: /opt/certs/ca-config.json
server 表示服务端连接客户端时携带的证书，用于客户端验证服务端身份
client 表示客户端连接服务端时携带的证书，用于服务端验证客户端身份
peer 表示相互之间连接时使用的证书，如etcd节点之间验证
[root@wang-200 ~]# cd /opt/certs/
[root@wang-200 certs]# cat >> /opt/certs/ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "175200h"
},
"profiles": {
"server": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth"
]
},
"client": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
# 创建etcd证书配置：/opt/certs/etcd-peer-csr.json，重点在hosts上，将所有可能的etcd服务器添加到host列表，不能使用网段，新增etcd服务器需要重新签发证书
[root@wang-200 certs]# cat >>/opt/certs/etcd-peer-csr.json <<EOF
{ "CN": "k8s-etcd", "hosts": [ "10.0.0.17","10.0.0.27","10.0.0.37","10.0.0.47" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "beijing", "L": "beijing", "O": "Ali", "OU": "ops" } ] }
EOF
# 签发证书
[root@wang-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer
[root@wang-200 certs]# ll etcd-peer*
-rw-r--r-- 1 root root 1070 Nov 25 20:54 etcd-peer.csr
-rw-r--r-- 1 root root 367 Nov 25 20:54 etcd-peer-csr.json
-rw------- 1 root root 1675 Nov 25 20:54 etcd-peer-key.pem
-rw-r--r-- 1 root root 1440 Nov 25 20:54 etcd-peer.pem
7-2、安装etcd
温馨提示：wang-27、wang-37、wang-47节点执行，建议下载3.1.20版本etcd，本人测试3.5版本etcd在supervisor拉不起来
etcd下载地址：https://github.com/etcd-io/etcd/
[root@wang-47 ~]# mkdir /opt/{apps,release,src}
[root@wang-47 ~]# cd /opt/src
[root@wang-47 src]# wget https://github.com/etcd-io/etcd/releases/download/v3.1.20/etcd-v3.1.20-linux-amd64.tar.gz
[root@wang-47 src]# tar xf etcd-v3.1.20-linux-amd64.tar.gz
[root@wang-47 src]# mv etcd-v3.1.20-linux-amd64 /opt/release/etcd-v3.1.20
[root@wang-47 src]# ln -s /opt/release/etcd-v3.1.20/ /opt/apps/etcd
[root@wang-47 src]# mkdir -p /opt/apps/etcd/certs /data/etcd /data/logs/etcd-server
[root@wang-47 src]# useradd -M -s /sbin/nologin etcd温馨提示：wang-200节点执行
# 下发证书到各个etcd上
[root@wang-200 certs]# for i in 27 37 47 ; do scp ca.pem etcd-peer.pem etcd-peer-key.pem wang-$i:/opt/apps/etcd/certs/; done
[root@wang-200 certs]# md5sum ca.pem etcd-peer.pem etcd-peer-key.pem
37038856139f2a771783b41de9b4ccee ca.pem
52afd068b8f32771ede7704f4539adab etcd-peer.pem
138678e1d2843af5376c997b2305e403 etcd-peer-key.pem
温馨提示：wang-27、wang-37、wang-47节点执行
[root@wang-47 src]# md5sum /opt/apps/etcd/certs/*
37038856139f2a771783b41de9b4ccee /opt/apps/etcd/certs/ca.pem
138678e1d2843af5376c997b2305e403 /opt/apps/etcd/certs/etcd-peer-key.pem
52afd068b8f32771ede7704f4539adab /opt/apps/etcd/certs/etcd-peer.pem
# 创建启动脚本
# 说明：
# listen-peer-urls etcd节点之间通信端口
# listen-client-urls 客户端与etcd通信端口
# quota-backend-bytes 配额大小
# 需要修改的参数（各节点都不一样）：name,listen-peer-urls,listen-client-urls,initial-advertise-peer-urls
[root@wang-47 src]# vim /opt/apps/etcd/etcd-server-startup.sh
#!/bin/sh
./etcd --name etcd-server-47 \
--data-dir /data/etcd/etcd-server \
--listen-peer-urls https://10.0.0.47:2380 \
--listen-client-urls https://10.0.0.47:2379,http://127.0.0.1:2379 \
--quota-backend-bytes 8000000000 \
--initial-advertise-peer-urls https://10.0.0.47:2380 \
--advertise-client-urls https://10.0.0.47:2379,http://127.0.0.1:2379 \
--initial-cluster etcd-server-47=https://10.0.0.47:2380,etcd-server-37=https://10.0.0.37:2380,etcd-server-27=https://10.0.0.27:2380 \
--ca-file ./certs/ca.pem \
--cert-file ./certs/etcd-peer.pem \
--key-file ./certs/etcd-peer-key.pem \
--client-cert-auth \
--trusted-ca-file ./certs/ca.pem \
--peer-ca-file ./certs/ca.pem \
--peer-cert-file ./certs/etcd-peer.pem \
--peer-key-file ./certs/etcd-peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file ./certs/ca.pem \
--log-output stdout
[root@wang-47 src]# chmod u+x /opt/apps/etcd/etcd-server-startup.sh
[root@wang-47 src]# chown -R etcd.etcd /opt/apps/etcd/ /data/etcd /data/logs/etcd-server/
# 启动etcd
# 因为这些进程都是要启动为后台进程，要么手动启动，要么采用后台进程管理工具，这里使用后台管理工具
[root@wang-47 src]# yum install -y supervisor
[root@wang-47 src]# systemctl start supervisord
[root@wang-47 src]# systemctl enable supervisord
[root@wang-47 src]# systemctl status supervisord
#说明：
# 各节点program不一样，其他都一样
# killasgroup=true #主要用于，supervisord管理的子进程，这个子进程本身还有，子进程。那么我们如果仅仅干掉supervisord的子进程的话，子进程的子进程，有可能会变成孤儿进程。所以咱们可以设置可个选项，把整个该子进程的，整个进程组都干掉。 设置为true的话，一般killasgroup也会被设置为true。需要注意的是，该选项发送的是stop信号，默认为false。。非必须设置。
# stopasgroup=true #和上面的stopasgroup类似，不过发送的是kill信号
[root@wang-47 src]# vim /etc/supervisord.d/etcd-server.ini
[program:etcd-server-47]
command=/opt/apps/etcd/etcd-server-startup.sh
numprocs=1
directory=/opt/apps/etcd
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=etcd
redirect_stderr=true
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=5
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
killasgroup=true
stopasgroup=true
[root@wang-47 src]# supervisorctl update
etcd-server-0-47: added process group
[root@wang-47 src]# supervisorctl status
etcd-server-47 FATAL Exited too quickly (process log may have details)
#需要restart才能起来
[root@wang-47 src]# supervisorctl restart etcd-server-47
etcd-server-47: stopped
etcd-server-47: started
[root@wang-47 src]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 0:00:46
# 3台均启动后，检查集群状态
[root@wang-47 etcd]# /opt/apps/etcd/etcdctl cluster-health
member 2ff8ca0b39f4ce02 is healthy: got healthy result from http://127.0.0.1:2379
member 36baf8e1cef98b33 is healthy: got healthy result from http://127.0.0.1:2379
member 6dd0bcb35d4c93f0 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@wang-47 etcd]# /opt/apps/etcd/etcdctl member list
2ff8ca0b39f4ce02: name=etcd-server-37 peerURLs=https://10.0.0.37:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.37:2379 isLeader=false
36baf8e1cef98b33: name=etcd-server-27 peerURLs=https://10.0.0.27:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.27:2379 isLeader=false
6dd0bcb35d4c93f0: name=etcd-server-47 peerURLs=https://10.0.0.47:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.47:2379 isLeader=true
# 说明：
# supervisor管理etcd启停方式
supervisorctl start etcd-server-47 #启动
supervisorctl stop etcd-server-47 #停止
supervisorctl restart etcd-server-47 #重启
supervisorctl status etcd-server-47 #查看状态
8、部署apiserver
8-1、下载kubernetes服务端
温馨提示：wang-37、wang-47节点执行
二进制下载地址: https://dl.k8s.io/v1.15.2/kubernetes-server-linux-amd64.tar.gz
[root@wang-47 src]# tar xf kubernetes-server-linux-amd64.tar.gz
[root@wang-47 src]# mv kubernetes /opt/release/kubernetes-v1.15.2
[root@wang-47 src]# ln -s /opt/release/kubernetes-v1.15.2/ /opt/apps/kubernetes
[root@wang-47 src]# cd /opt/apps/kubernetes
[root@wang-47 kubernetes]# rm -f kubernetes-src.tar.gz #这是源码包，删掉
[root@wang-47 kubernetes]# cd server/bin/
[root@wang-47 bin]# rm -f *.tar *_tag #这些是kubeadm安装所需的镜像包，删掉
[root@wang-47 bin]# ll
total 545892
-rwxr-xr-x 1 root root 48123904 Jun 17 2020 apiextensions-apiserver
-rwxr-xr-x 1 root root 39809024 Jun 17 2020 kubeadm
-rwxr-xr-x 1 root root 120664064 Jun 17 2020 kube-apiserver
-rwxr-xr-x 1 root root 110055424 Jun 17 2020 kube-controller-manager
-rwxr-xr-x 1 root root 44032000 Jun 17 2020 kubectl
-rwxr-xr-x 1 root root 113283800 Jun 17 2020 kubelet
-rwxr-xr-x 1 root root 38383616 Jun 17 2020 kube-proxy
-rwxr-xr-x 1 root root 42950656 Jun 17 2020 kube-scheduler
-rwxr-xr-x 1 root root 1687552 Jun 17 2020 mounter
8-2、签发证书
温馨提示：wang-200节点执行
apiserver与etcd进行通信时，apiserver是客户端，etcd是服务端，因此需要client证书
# 签发client证书（apiserver和etcd通信证书）
[root@wang-200 certs]# cat>> /opt/certs/client-csr.json <<EOF
{
"CN": "k8s-node",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
[root@wang-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json |cfssl-json -bare client
其他客户端来找apiserver时也需要ssl认证，生成apiserver的server端证书
# 签发server证书（apiserver和其它k8s组件通信使用）
# hosts中将所有可能作为apiserver的ip添加进去，VIP 10.0.0.99也要加
[root@wang-200 certs]# cat >> /opt/certs/apiserver-csr.json <<EOF
{
"CN": "k8s-apiserver",
"hosts": [
"127.0.0.1",
"192.168.0.1",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"10.0.0.99",
"10.0.0.37",
"10.0.0.47",
"10.0.0.57"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
[root@wang-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json |cfssl-json -bare apiserver
# 下发证书
[root@wang-200 certs]# for i in 37 47;do echo wang7-$i;ssh wang-$i "mkdir /opt/apps/kubernetes/server/bin/certs";scp apiserver-key.pem apiserver.pem ca-key.pem ca.pem client-key.pem client.pem wang-$i:/opt/apps/kubernetes/server/bin/certs/;done
8-3、配置apiserver日志审计
温馨提示：wang-37、wang-47节点执行
[root@wang-47 src]# mkdir /opt/apps/kubernetes/conf
[root@wang-47 src]# vim /opt/apps/kubernetes/conf/audit.yaml
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
resources:
- group: ""
# Resource "pods" doesn't match requests to any subresource of pods,
# which is consistent with the RBAC policy.
resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
resources:
- group: ""
resources: ["pods/log", "pods/status"]
# Don't log requests to a configmap called "controller-leader"
- level: None
resources:
- group: ""
resources: ["configmaps"]
resourceNames: ["controller-leader"]
# Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core API group
resources: ["endpoints", "services"]
# Don't log authenticated requests to certain non-resource URL paths.
- level: None
userGroups: ["system:authenticated"]
nonResourceURLs:
- "/api*" # Wildcard matching.
- "/version"
# Log the request body of configmap changes in kube-system.
- level: Request
resources:
- group: "" # core API group
resources: ["configmaps"]
# This rule only applies to resources in the "kube-system" namespace.
# The empty string "" can be used to select non-namespaced resources.
namespaces: ["kube-system"]
# Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
resources:
- group: "" # core API group
resources: ["secrets", "configmaps"]
# Log all other resources in core and extensions at the Request level.
- level: Request
resources:
- group: "" # core API group
- group: "extensions" # Version of group should NOT be included.
# A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
# Long-running requests like watches that fall under this rule will not
# generate an audit event in RequestReceived.
omitStages:
- "RequestReceived"
8-4、创建启动脚本
温馨提示：wang-37、wang-47节点执行
# 创建启动脚本
[root@wang-47 src]# vim /opt/apps/kubernetes/server/bin/kube-apiserver-startup.sh
#!/bin/bash
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/kubernetes/server/bin/kube-apiserver \
--apiserver-count 2 \
--audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
--audit-policy-file ../../conf/audit.yaml \
--authorization-mode RBAC \
--client-ca-file ./certs/ca.pem \
--requestheader-client-ca-file ./certs/ca.pem \
--enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
--etcd-cafile ./certs/ca.pem \
--etcd-certfile ./certs/client.pem \
--etcd-keyfile ./certs/client-key.pem \
--etcd-servers https://10.0.0.27:2379,https://10.0.0.37:2379,https://10.0.0.47:2379 \
--service-account-key-file ./certs/ca-key.pem \
--service-cluster-ip-range 192.168.0.0/16 \
--service-node-port-range 3000-29999 \
--target-ram-mb=1024 \
--kubelet-client-certificate ./certs/client.pem \
--kubelet-client-key ./certs/client-key.pem \
--log-dir /data/logs/kubernetes/kube-apiserver \
--tls-cert-file ./certs/apiserver.pem \
--tls-private-key-file ./certs/apiserver-key.pem \
--v 2
[root@wang-47 src]# chmod +x /opt/apps/kubernetes/server/bin/kube-apiserver-startup.sh
# 配置supervisor启动配置
[root@wang-47 src]# mkdir /data/logs/kubernetes/kube-apiserver -p
[root@wang-47 src]# vim /etc/supervisord.d/kube-apiserver.ini
[program:kube-apiserver-47]
command=/opt/apps/kubernetes/server/bin/kube-apiserver-startup.sh
numprocs=1
directory=/opt/apps/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=5
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
killasgroup=true
stopasgroup=true
[root@wang-47 src]# supervisorctl status
[root@wang-47 src]# supervisorctl restart kube-apiserver-47
[root@wang-47 src]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 3:59:24
kube-apiserver-47 RUNNING pid 5484, uptime 1:42:19
# 说明：
# supervisor管理kube-apiserver启停方式
supervisorctl start kube-apiserver-47 #启动
supervisorctl stop kube-apiserver-47 #停止
supervisorctl restart kube-apiserver-47 #重启
supervisorctl status kube-apiserver-47 #查看状态
9、部署keepalived+nginx反向代理api-server
温馨提示：wang-17、wang-27节点执行
9-1、部署nginx
[root@wang-17 ~]# yum install -y nginx nginx-all-modules.noarch
# 最后增加（日志格式、四层负载）
[root@wang-17 ~]# vim /etc/nginx/nginx.conf
stream {
log_format proxy '$time_local|$remote_addr|$upstream_addr|$protocol|$status|'
'$session_time|$upstream_connect_time|$bytes_sent|$bytes_received|'
'$upstream_bytes_sent|$upstream_bytes_received';
upstream kube-apiserver {
server 10.0.0.37:6443 max_fails=3 fail_timeout=30s;
server 10.0.0.47:6443 max_fails=3 fail_timeout=30s;
}
server {
listen 7443;
proxy_connect_timeout 2s;
proxy_timeout 900s;
proxy_pass kube-apiserver;
access_log /var/log/nginx/proxy.log proxy;
}
}
[root@wang-17 ~]# nginx -t
[root@wang-17 ~]# systemctl start nginx
[root@wang-17 ~]# systemctl enable nginx
[root@wang-17 ~]# systemctl status nginx
# 测试几次，查看日志格式
[root@wang-17 ~]# curl 127.0.0.1:7443
Client sent an HTTP request to an HTTPS server.
[root@wang-17 ~]# cat /var/log/nginx/proxy.log
26/Nov/2022:14:27:03 +0800|127.0.0.1|10.0.0.37:6443|TCP|200|0.001|0.001|76|78|78|76
26/Nov/2022:14:27:06 +0800|127.0.0.1|10.0.0.37:6443|TCP|200|0.002|0.000|76|78|78|76
26/Nov/2022:14:27:10 +0800|127.0.0.1|10.0.0.47:6443|TCP|200|0.001|0.001|76|78|78|76
26/Nov/2022:14:27:11 +0800|127.0.0.1|10.0.0.37:6443|TCP|200|0.002|0.001|76|78|78|76
26/Nov/2022:14:27:11 +0800|127.0.0.1|10.0.0.47:6443|TCP|200|0.001|0.000|76|78|78|76
26/Nov/2022:14:27:11 +0800|127.0.0.1|10.0.0.47:6443|TCP|200|0.001|0.001|76|78|78|76
9-2、部署keepalived
# 说明：
#keepalived 监控端口脚本
#使用方法：
#在keepalived的配置文件中
#vrrp_script check_port {#创建一个vrrp_script脚本,检查配置
# script "/etc/keepalived/check_port.sh 7443" #配置监听的端口
# interval 2 #检查脚本的频率,单位（秒）
#}
[root@wang-17 ~]# vim /etc/keepalived/check_port.sh
#!/bin/bash
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
PORT_PROCESS=`ss -lnt|grep $CHK_PORT|wc -l`
if [ $PORT_PROCESS -eq 0 ];then
echo "Port $CHK_PORT Is Not Used,End."
exit 1
fi
else
echo "Check Port Cant Be Empty!"
fi
[root@wang-17 ~]# chmod +x /etc/keepalived/check_port.sh
温馨提示：
主节点中，必须加上 nopreempt
因为一旦因为网络抖动导致VIP漂移，不能让它自动飘回来，必须要分析原因后手动迁移VIP到主节点！如主节点确认正常后，重启备节点的keepalive，让VIP飘到主节点.
keepalived 的日志输出配置此处省略，生产中需要进行处理。
# 说明：
# wang-17为keepalived主，wang-27为备份，vip地址是10.0.0.99
# 根据实际的网卡名称更改网卡名配置
# 此处使用的是单播，也建议使用单播，因为如果虚拟实例多的话，多播会导致vip漂移失败。
# keepalived主 配置
[root@wang-17 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id 10.0.0.17
}
vrrp_script chk_nginx{
script "/etc/keepalived/check_port.sh 7443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 99
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.99
}
track_script {
chk_nginx
}
unicast_src_ip 10.0.0.17
unicast_peer {
10.0.0.27
}
}
# keepalived备 配置
[root@wang-27 ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id 10.0.0.27
}
vrrp_script chk_nginx{
script "/etc/keepalived/check_port.sh 7443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 99
priority 80
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.99
}
track_script {
chk_nginx
}
unicast_src_ip 10.0.0.27
unicast_peer {
10.0.0.17
}
}
# 启动keepalived
[root@wang-17 ~]# systemctl start keepalived && systemctl enable keepalived && systemctl status keepalived
10、部署controller-manager
温馨提示：wang-37、wang-47节点执行
kube-controller-manager组件只是跟本机部署的api-server进行通信，因此不用ssl加密，因此不需要签发证书，一般生产建议这样配置。但如果想让kube-controller-manager跟集群所有的其他组件ssl加密，就需要同样签发两套证书，参考api-server签发证书配置
# 创建启动脚本
[root@wang-47 src]# vim /opt/apps/kubernetes/server/bin/kube-controller-manager-startup.sh
#!/bin/sh
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/kubernetes/server/bin/kube-controller-manager \
--cluster-cidr 172.7.0.0/16 \
--leader-elect true \
--log-dir /data/logs/kubernetes/kube-controller-manager \
--master http://127.0.0.1:8080 \
--service-account-private-key-file ./certs/ca-key.pem \
--service-cluster-ip-range 192.168.0.0/16 \
--root-ca-file ./certs/ca.pem \
--v 2
[root@wang-47 src]# chmod +x /opt/apps/kubernetes/server/bin/kube-controller-manager-startup.sh
[root@wang-47 src]# mkdir /data/logs/kubernetes/kube-controller-manager
# 配置supervisor启动配置
[root@wang-47 src]# vim /etc/supervisord.d/kube-controller-manager.ini
[program:kube-controller-manager-47]
command=/opt/apps/kubernetes/server/bin/kube-controller-manager-startup.sh
numprocs=1
directory=/opt/apps/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controller.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
killasgroup=true
stopasgroup=true
# 启动服务并检查
[root@wang-47 src]# supervisorctl update
[root@wang-47 src]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 5:17:10
kube-apiserver-47 RUNNING pid 5484, uptime 3:00:05
kube-controller-manager-47 RUNNING pid 5613, uptime 0:02:33
11、部署scheduler
温馨提示：wang-37、wang-47节点执行
kube-scheduler组件只是跟本机部署的api-server进行通信，因此不用ssl加密，因此不需要签发证书，一般生产建议这样配置。但如果想让kube-scheduler跟集群所有的其他组件ssl加密，就需要同样签发两套证书，参考api-server签发证书配置
# 创建启动脚本
[root@wang-47 src]# vim /opt/apps/kubernetes/server/bin/kube-scheduler-startup.sh
#!/bin/sh
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/kubernetes/server/bin/kube-scheduler \
--leader-elect \
--log-dir /data/logs/kubernetes/kube-scheduler \
--master http://127.0.0.1:8080 \
--v 2
[root@wang-47 src]# chmod +x /opt/apps/kubernetes/server/bin/kube-scheduler-startup.sh
[root@wang-47 src]# mkdir /data/logs/kubernetes/kube-scheduler
# 配置supervisor启动配置
[root@wang-47 src]# vim /etc/supervisord.d/kube-scheduler.ini
[program:kube-scheduler-47]
command=/opt/apps/kubernetes/server/bin/kube-scheduler-startup.sh
numprocs=1
directory=/opt/apps/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
killasgroup=true
stopasgroup=true
# 启动服务并检查
[root@wang-47 src]# supervisorctl update
[root@wang-47 src]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 5:31:14
kube-apiserver-47 RUNNING pid 5484, uptime 3:14:09
kube-controller-manager-47 RUNNING pid 5613, uptime 0:16:37
kube-scheduler-47 RUNNING pid 5642, uptime 0:04:55
# 检查主控节点状态
[root@wang-47 src]# ln -s /opt/apps/kubernetes/server/bin/kubectl /usr/local/bin/
[root@wang-47 src]# kubectl get cs
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-1 Healthy {"health": "true"}
etcd-2 Healthy {"health": "true"}
etcd-0 Healthy {"health": "true"}
12、部署node节点的kubelet
12-1、签发证书
温馨提示：wang-200节点执行
# hosts字段把可能部署kubelet的主机ip都写进去，只能写ip地址，不能写网段。后期如果在非hosts列表中ip主机上部署kubelet，需要重新签发证书，并更换证书，并重启服务
[root@wang-200 certs]# vim kubelet-csr.json
{
"CN": "k8s-kubelet",
"hosts": [
"127.0.0.1",
"10.0.0.17",
"10.0.0.27",
"10.0.0.37",
"10.0.0.47",
"10.0.0.57",
"10.0.0.67",
"10.0.0.77",
"10.0.0.87",
"10.0.0.97"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "k8s",
"OU": "System"
}
]
}
[root@wang-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssl-json -bare kubelet
[root@wang-200 certs]# ll kubelet*
-rw-r--r-- 1 root root 1123 Nov 26 15:30 kubelet.csr
-rw-r--r-- 1 root root 456 Nov 26 15:30 kubelet-csr.json
-rw------- 1 root root 1675 Nov 26 15:30 kubelet-key.pem
-rw-r--r-- 1 root root 1468 Nov 26 15:30 kubelet.pem
[root@wang-200 certs]# for i in 37 47;do scp kubelet.pem kubelet-key.pem wang-$i:/opt/apps/kubernetes/server/bin/certs/;done
12-2、创建kubelet配置
温馨提示：wang-37、wang-47节点执行
# set-cluster 创建需要连接的集群信息，可以创建多个k8s集群信息
[root@wang-47 src]# kubectl config set-cluster myk8s \
--certificate-authority=/opt/apps/kubernetes/server/bin/certs/ca.pem \
--embed-certs=true \
--server=https://10.0.0.99:7443 \
--kubeconfig=/opt/apps/kubernetes/conf/kubelet.kubeconfig
# set-credentials 创建用户账号，即用户登陆使用的客户端私有和证书，可以创建多个证书
[root@wang-47 src]# kubectl config set-credentials k8s-node \
--client-certificate=/opt/apps/kubernetes/server/bin/certs/client.pem \
--client-key=/opt/apps/kubernetes/server/bin/certs/client-key.pem \
--embed-certs=true \
--kubeconfig=/opt/apps/kubernetes/conf/kubelet.kubeconfig
# set-context 设置context，即确定账号和集群对应关系
[root@wang-47 src]# kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=k8s-node \
--kubeconfig=/opt/apps/kubernetes/conf/kubelet.kubeconfig
# use-context 设置当前使用哪个context
[root@wang-47 src]# kubectl config use-context myk8s-context --kubeconfig=/opt/apps/kubernetes/conf/kubelet.kubeconfig
12-3、授权k8s-node用户
此步骤只需要在一台master节点执行
授权 k8s-node 用户绑定集群角色 system:node ，让 k8s-node 成为具备运算节点的权限。
[root@wang-47 conf]# vim k8s-node.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: k8s-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: k8s-node
#根据rbac规格给k8s-node用户授予权限：集群里成为运算节点的权限，此资源已经写入etcd中，不需要在其他节点重复执行了
[root@wang-47 conf]# kubectl create -f k8s-node.yaml
[root@wang-47 conf]# kubectl get clusterrolebinding k8s-node
NAME AGE
k8s-node 16s
12-4、准备pause基础镜像
温馨提示：wang-200节点执行
准备pause基础镜像，并将pause镜像放入到harbor私有仓库中
[root@wang-200 ~]# docker pull kubernetes/pause
[root@wang-200 ~]# docker tag kubernetes/pause:latest harbor.wang.com/public/pause:latest
[root@wang-200 ~]# docker login harbor.wang.com
[root@wang-200 ~]# docker push harbor.wang.com/public/pause:latest
12-5、创建启动脚本
温馨提示：wang-37、wang-47节点执行
kubelet集群各主机的启动脚本略有不同，部署节点时注意修改
# 创建启动脚本
[root@wang-47 conf]# vim /opt/apps/kubernetes/server/bin/kubelet-startup.sh
#!/bin/sh
# 注意修改主机名
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/kubernetes/server/bin/kubelet \
--anonymous-auth=false \
--cgroup-driver systemd \
--cluster-dns 192.168.0.2 \
--cluster-domain cluster.local \
--runtime-cgroups=/systemd/system.slice \
--kubelet-cgroups=/systemd/system.slice \
--fail-swap-on="false" \
--client-ca-file ./certs/ca.pem \
--tls-cert-file ./certs/kubelet.pem \
--tls-private-key-file ./certs/kubelet-key.pem \
--hostname-override wang-47.host.com \
--image-gc-high-threshold 20 \
--image-gc-low-threshold 10 \
--kubeconfig ../../conf/kubelet.kubeconfig \
--log-dir /data/logs/kubernetes/kube-kubelet \
--pod-infra-container-image harbor.wang.com/public/pause:latest \
--root-dir /data/kubelet
[root@wang-47 conf]# chmod +x /opt/apps/kubernetes/server/bin/kubelet-startup.sh
[root@wang-47 conf]# mkdir -p /data/logs/kubernetes/kube-kubelet /data/kubelet
# 配置supervisor启动配置
[root@wang-47 conf]# vim /etc/supervisord.d/kube-kubelet.ini
[program:kube-kubelet-47]
command=/opt/apps/kubernetes/server/bin/kubelet-startup.sh
numprocs=1
directory=/opt/apps/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=5
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
killasgroup=true
stopasgroup=true
# 启动服务并检查
[root@wang-47 conf]# supervisorctl update
[root@wang-47 conf]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 6:56:05
kube-apiserver-47 RUNNING pid 5484, uptime 4:39:00
kube-controller-manager-47 RUNNING pid 5613, uptime 1:41:28
kube-kubelet-47 RUNNING pid 5761, uptime 0:01:04
kube-scheduler-47 RUNNING pid 5642, uptime 1:29:46
[root@wang-47 conf]# kubectl get node
NAME STATUS ROLES AGE VERSION
wang-37.host.com Ready <none> 94s v1.15.2
wang-47.host.com Ready <none> 4m13s v1.15.2
12-6、修改节点角色label
使用 kubectl get nodes 获取的Node节点角色为空，可以按照以下方式修改,这个操作只是在节点上面打了个标签。
[root@wang-47 conf]# kubectl label node wang-37.host.com node-role.kubernetes.io/node=
[root@wang-47 conf]# kubectl label node wang-37.host.com node-role.kubernetes.io/master=
[root@wang-47 conf]# kubectl label node wang-47.host.com node-role.kubernetes.io/master=
[root@wang-47 conf]# kubectl label node wang-47.host.com node-role.kubernetes.io/node=
[root@wang-47 conf]# kubectl get node
NAME STATUS ROLES AGE VERSION
wang-37.host.com Ready master,node 7m52s v1.15.2
wang-47.host.com Ready master,node 10m v1.15.2
13、部署kube-proxy
13-1、签发证书
温馨提示：wang-200节点执行
# 因为kube-proxy使用的用户是kube-proxy，不能使用client证书，必须要重新签发自己的证书
[root@wang-200 certs]# cat >> kube-proxy-csr.json <<EOF
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
[root@wang-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssl-json -bare kube-proxy-client
[root@wang-200 certs]# for i in 37 47;do scp kube-proxy-client-key.pem kube-proxy-client.pem wang-$i:/opt/apps/kubernetes/server/bin/certs/;done
13-2、创建kube-proxy配置
温馨提示：wang-37、wang-47节点执行
# set-cluster
[root@wang-47 conf]# kubectl config set-cluster myk8s \
--certificate-authority=/opt/apps/kubernetes/server/bin/certs/ca.pem \
--embed-certs=true \
--server=https://10.0.0.99:7443 \
--kubeconfig=/opt/apps/kubernetes/conf/kube-proxy.kubeconfig
# set-credentials
[root@wang-47 conf]# kubectl config set-credentials kube-proxy \
--client-certificate=/opt/apps/kubernetes/server/bin/certs/kube-proxy-client.pem \
--client-key=/opt/apps/kubernetes/server/bin/certs/kube-proxy-client-key.pem \
--embed-certs=true \
--kubeconfig=/opt/apps/kubernetes/conf/kube-proxy.kubeconfig
# set-context
[root@wang-47 conf]# kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=kube-proxy \
--kubeconfig=/opt/apps/kubernetes/conf/kube-proxy.kubeconfig
# use-context
[root@wang-47 conf]# kubectl config use-context myk8s-context --kubeconfig=/opt/apps/kubernetes/conf/kube-proxy.kubeconfig
13-3、加载ipvs模块
温馨提示：wang-37、wang-47节点执行
kube-proxy 共有3种流量调度模式，分别是 namespace，iptables，ipvs，其中ipvs性能最好
[root@wang-47 ~]# vim /root/ipvs.sh
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for i in $(ls $ipvs_mods_dir|grep -o "^[^.]*")
do
/sbin/modinfo -F filename $i &>/dev/null
if [ $? -eq 0 ];then
/sbin/modprobe $i
fi
done
[root@wang-47 ~]# lsmod | grep ip_vs
[root@wang-47 ~]# chmod +x /root/ipvs.sh
[root@wang-47 ~]# bash /root/ipvs.sh
[root@wang-47 ~]# lsmod | grep ip_vs
13-4、创建启动脚本
温馨提示：wang-37、wang-47节点执行
# 创建启动脚本
# 注意更改主机名
[root@wang-47 ~]# vim /opt/apps/kubernetes/server/bin/kube-proxy-startup.sh
#!/bin/sh
WORK_DIR=$(dirname $(readlink -f $0))
[ $? -eq 0 ] && cd $WORK_DIR || exit
/opt/apps/kubernetes/server/bin/kube-proxy \
--cluster-cidr 172.7.0.0/16 \
--hostname-override wang-47.host.com \
--proxy-mode=ipvs \
--ipvs-scheduler=nq \
--kubeconfig ../../conf/kube-proxy.kubeconfig
[root@wang-47 ~]# chmod +x /opt/apps/kubernetes/server/bin/kube-proxy-startup.sh
[root@wang-47 ~]# mkdir -p /data/logs/kubernetes/kube-proxy
# 配置supervisor启动配置
[root@wang-47 ~]# vim /etc/supervisord.d/kube-proxy.ini
[program:kube-proxy-47]
command=/opt/apps/kubernetes/server/bin/kube-proxy-startup.sh
numprocs=1
directory=/opt/apps/kubernetes/server/bin
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=5
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
killasgroup=true
stopasgroup=true
# 启动服务并检查
[root@wang-47 ~]# supervisorctl update
[root@wang-47 ~]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 7:30:25
kube-apiserver-47 RUNNING pid 5484, uptime 5:13:20
kube-controller-manager-47 RUNNING pid 5613, uptime 2:15:48
kube-kubelet-47 RUNNING pid 5761, uptime 0:35:24
kube-proxy-47 RUNNING pid 11545, uptime 0:01:34
kube-scheduler-47 RUNNING pid 5642, uptime 2:04:06
14、验证集群
在任意一个运算节点，创建一个资源配置清单
[root@wang-47 ~]# vim /root/nginx-ds.yaml
metadata:
name: nginx-ds
spec:
template:
metadata:
labels:
app: nginx-ds
spec:
containers:
- name: my-nginx
image: harbor.wang.com/public/nginx:1.22
ports:
- containerPort: 80
[root@wang-47 ~]# kubectl apply -f /root/nginx-ds.yaml
root@wang-47 ~]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-ds-cpm56 1/1 Running 0 28s 172.7.47.2 wang-47.host.com <none> <none>
nginx-ds-d92h2 1/1 Running 0 28s 172.7.37.2 wang-37.host.com <none> <none>
[root@wang-47 ~]# curl -I 172.7.47.2
HTTP/1.1 200 OK
Server: nginx/1.22.1
Date: Sat, 26 Nov 2022 09:26:34 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Wed, 19 Oct 2022 08:02:20 GMT
Connection: keep-alive
ETag: "634faf0c-267"
Accept-Ranges: bytes
[root@wang-47 ~]# curl -i 172.7.37.2
curl: (56) Recv failure: Connection reset by peer
# 可以看到已经起了两个pod，使用curl命令只能通一个，是因为跨宿主机之间还不能通信。
15、命令补全
温馨提示：wang-37、wang-47节点执行
[root@wang-47 ~]# yum install bash-completion -y
[root@wang-47 ~]# source /usr/share/bash-completion/bash_completion
[root@wang-47 ~]# source <(kubectl completion bash)
[root@wang-47 ~]# echo "source <(kubectl completion bash)" >> ~/.bashrc
16、部署CNI插件flannel
温馨提示：wang-37、wang-47节点执行
[root@wang-47 src]# wget https://github.com/flannel-io/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
[root@wang-47 src]# mkdir /opt/flannel-v0.11.0
[root@wang-47 src]# tar xf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0/
[root@wang-47 src]# ln -s /opt/flannel-v0.11.0/ /opt/flannel
[root@wang-47 src]# mkdir /opt/flannel/certs
温馨提示：wang-200节点执行
flannel默认是需要使用etcd做一些存储和配置的，因此需要flannel使用client证书连接etcd
[root@wang-200 certs]# scp ca.pem client.pem client-key.pem 10.0.0.47:/opt/flannel/certs/
[root@wang-200 certs]# scp ca.pem client.pem client-key.pem 10.0.0.37:/opt/flannel/certs/
温馨提示：wang-37、wang-47节点执行
# 第一行表示管理的网络，第二行表示本机的网络，在其他节点上进行配置时需要进行修改
[root@wang-47 src]# vim /opt/flannel/subnet.env
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.47.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
# 注意：ip写本机ip，iface写通信的接口名，根据实际进行修改，不同节点配置略有不同
[root@wang-47 src]# vim /opt/flannel/flanneld.sh
#!/bin/sh
./flanneld \
--public-ip=10.0.0.47 \
--etcd-endpoints=https://10.0.0.47:2379,https://10.0.0.37:2379,https://10.0.0.27:2379 \
--etcd-keyfile=./certs/client-key.pem \
--etcd-certfile=./certs/client.pem \
--etcd-cafile=./certs/ca.pem \
--iface=eth0 \
--subnet-file=./subnet.env \
--healthz-port=2401
[root@wang-47 src]# chmod +x /opt/flannel/flanneld.sh
# 注意：这一步只需要做一次就行，不需要在其他节点重复配置了
[root@wang-47 src]# cd /opt/apps/etcd
[root@wang-47 etcd]# ./etcdctl member list
2ff8ca0b39f4ce02: name=etcd-server-37 peerURLs=https://10.0.0.37:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.37:2379 isLeader=false
36baf8e1cef98b33: name=etcd-server-27 peerURLs=https://10.0.0.27:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.27:2379 isLeader=false
6dd0bcb35d4c93f0: name=etcd-server-47 peerURLs=https://10.0.0.47:2380 clientURLs=http://127.0.0.1:2379,https://10.0.0.47:2379 isLeader=true
# 因为flannel需要etcd做一些存储配置，因此需要在etcd中创建有关flannel的配置
[root@wang-47 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
[root@wang-47 etcd]# ./etcdctl get /coreos.com/network/config
{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}
# flannel依托于端口转发功能，每台安装flannel的服务器必须配置端口转发
[root@wang-47 etcd]# vim /etc/sysctl.conf
......
net.ipv4.ip_forward = 1
[root@wang-47 etcd]# sysctl -p
# 创建supervisor配置
[root@wang-47 etcd]# vim /etc/supervisord.d/flannel.ini
[program:flanneld-47]
command=/opt/flannel/flanneld.sh
numprocs=1
directory=/opt/flannel
autostart=true
autorestart=true
startsecs=30
startretries=3
exitcodes=0,2
stopsignal=QUIT
stopwaitsecs=10
user=root
redirect_stderr=true
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log
stdout_logfile_maxbytes=64MB
stdout_logfile_backups=4
stdout_capture_maxbytes=1MB
stdout_events_enabled=false
# 启动服务并检查
[root@wang-47 etcd]# supervisorctl update
[root@wang-47 etcd]# supervisorctl status
[root@wang-47 etcd]# supervisorctl status
etcd-server-47 RUNNING pid 5101, uptime 10:08:12
flanneld-47 RUNNING pid 36367, uptime 1:01:37
kube-apiserver-47 RUNNING pid 5484, uptime 7:51:07
kube-controller-manager-47 RUNNING pid 5613, uptime 4:53:35
kube-kubelet-47 RUNNING pid 5761, uptime 3:13:11
kube-proxy-47 RUNNING pid 11545, uptime 2:39:21
kube-scheduler-47 RUNNING pid 5642, uptime 4:41:53
[root@wang-47 etcd]# kubectl get pod -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-ds-cpm56 1/1 Running 0 3h4m 172.7.47.2 wang-47.host.com <none> <none>
nginx-ds-d92h2 1/1 Running 0 3h4m 172.7.37.2 wang-37.host.com <none> <none>
# 通过ip可以看到这两个pod分别在两个不同的宿主机上部署着，之前不能curl通其他宿主机上的pod，现在已经可以了
[root@wang-47 etcd]# ping -c1 172.7.47.2
PING 172.7.47.2 (172.7.47.2) 56(84) bytes of data.
64 bytes from 172.7.47.2: icmp_seq=1 ttl=64 time=0.373 ms
--- 172.7.47.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.373/0.373/0.373/0.000 ms
[root@wang-47 etcd]# ping -c1 172.7.37.2
PING 172.7.37.2 (172.7.37.2) 56(84) bytes of data.
64 bytes from 172.7.37.2: icmp_seq=1 ttl=63 time=1.01 ms
--- 172.7.37.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.019/1.019/1.019/0.000 ms
[root@wang-47 etcd]# curl -I 172.7.37.2
HTTP/1.1 200 OK
Server: nginx/1.22.1
Date: Sat, 26 Nov 2022 12:31:09 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Wed, 19 Oct 2022 08:02:20 GMT
Connection: keep-alive
ETag: "634faf0c-267"
Accept-Ranges: bytes
[root@wang-47 etcd]# curl -I 172.7.47.2
HTTP/1.1 200 OK
Server: nginx/1.22.1
Date: Sat, 26 Nov 2022 12:31:12 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Wed, 19 Oct 2022 08:02:20 GMT
Connection: keep-alive
ETag: "634faf0c-267"
Accept-Ranges: bytes
17、部署coredns
k8s里的DNS不是万能的，他应该只负责自动维护“服务名”-------> “集群网络IP”之间的关系
17-1、部署k8s的内网资源配置清单http服务
温馨提示：wang-200节点执行
在wang-200上，配置一个nginx虚拟主机，用来提供k8s统一的资源配置清单访问入口
[root@wang-200 ~]# vim /etc/nginx/conf.d/k8s-yaml.wang.com.conf
server {
listen 80;
server_name k8s-yaml.wang.com;
location / {
autoindex on;
default_type text/plain;
root /data/k8s-yaml;
}
}
[root@wang-200 ~]# nginx -t
[root@wang-200 ~]# nginx -s reload
[root@wang-200 ~]# mkdir /data/k8s-yaml
温馨提示：wang-17节点执行
# 配置内网dns解析
[root@wang-17 ~]# vim /var/named/wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112503 ; serial
108000 ; refresh
900 ; retry
604800 ; expire
86400 ) ; minimum
NS dns.wang.com.
dns A 10.0.0.17
harbor A 10.0.0.200
k8s-yaml A 10.0.0.200
[root@wang-17 ~]# systemctl restart named
[root@wang-17 ~]# systemctl status named
[root@wang-17 ~]# dig -t A k8s-yaml.wang.com @10.0.0.17 +short
10.0.0.200
17-2、下载coredns
温馨提示：wang-200节点执行
GitHub下载地址：https://github.com/coredns/coredns
Docker下载地址：https://hub.docker.com/r/coredns/coredns/tags
#下载coredns的镜像，并推送到harbor里面
[root@wang-200 ~]# docker pull coredns/coredns:1.6.1
[root@wang-200 ~]# docker tag coredns/coredns:1.6.1 harbor.wang.com/public/coredns:1.6.1
[root@wang-200 ~]# docker push harbor.wang.com/public/coredns:1.6.1
17-3、准备资源配置清单
温馨提示：wang-200节点执行
官方参考地址：https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/coredns/coredns.yaml.base
17-3-1、rbac资源配置清单
[root@wang-200 ~]# cd /data/k8s-yaml/
[root@wang-200 k8s-yaml]# mkdir coredns
[root@wang-200 k8s-yaml]# cd coredns
[root@wang-200 coredns]# vim /data/k8s-yaml/coredns/rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: Reconcile
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: EnsureExists
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
17-3-2、cm资源配置清单
[root@wang-200 coredns]# vim /data/k8s-yaml/coredns/cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
log
health
ready
kubernetes cluster.local 192.168.0.0/16
forward . 10.0.0.17
cache 30
loop
reload
loadbalance
}
17-3-3、dp资源配置清单
[root@wang-200 coredns]# vim /data/k8s-yaml/coredns/dp.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/name: "CoreDNS"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: coredns
template:
metadata:
labels:
k8s-app: coredns
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
containers:
- name: coredns
image: harbor.wang.com/public/coredns:1.6.1
args:
- -conf
- /etc/coredns/Corefile
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
17-3-4、svc资源配置清单
[root@wang-200 coredns]# vim /data/k8s-yaml/coredns/svc.yaml
apiVersion: v1
kind: Service
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: coredns
clusterIP: 192.168.0.2
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
- name: metrics
port: 9153
protocol: TCP
[root@wang-200 coredns]# ll /data/k8s-yaml/coredns/
total 16
-rw-r--r-- 1 root root 319 Nov 26 20:57 cm.yaml
-rw-r--r-- 1 root root 1295 Nov 26 21:00 dp.yaml
-rw-r--r-- 1 root root 954 Nov 26 20:54 rbac.yaml
-rw-r--r-- 1 root root 387 Nov 26 21:01 svc.yaml
17-4、创建资源
温馨提示：wang-47或wang-37其中一个节点执行
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/coredns/rbac.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/coredns/cm.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/coredns/dp.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/coredns/svc.yaml
# 验证
[root@wang-47 ~]# kubectl get all -n kube-system
NAME READY STATUS RESTARTS AGE
pod/coredns-567ff85894-jvqrg 1/1 Running 0 103s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/coredns ClusterIP 192.168.0.2 <none> 53/UDP,53/TCP,9153/TCP 93s
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/coredns 1/1 1 1 103s
NAME DESIRED CURRENT READY AGE
replicaset.apps/coredns-567ff85894 1 1 1 103s
[root@wang-47 ~]# dig -t A www.baidu.com @192.168.0.2 +short
www.a.shifen.com.
220.181.38.150
220.181.38.149
[root@wang-47 ~]# dig -t A wang-47.host.com @192.168.0.2 +short
10.0.0.47
[root@wang-47 ~]# dig -t A wang-17.host.com @192.168.0.2 +short
10.0.0.17
# 说明：在前面kubelet的启动脚本中已经制定了dns的ip是192.168.0.2，因此这里可以解析
[root@wang-47 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 192.168.0.1 <none> 443/TCP 9h
[root@wang-47 ~]# dig -t A kubernetes.default.svc.cluster.local. @192.168.0.2 +short
192.168.0.1
[root@wang-47 ~]# kubectl exec nginx-ds-cpm56 cat /etc/resolv.conf
nameserver 192.168.0.2
search default.svc.cluster.local svc.cluster.local cluster.local host.com
options ndots:5
18、部署ingress-traefik
18-1、下载traefik
温馨提示：wang-200节点执行
GitHub下载地址：https://github.com/traefik/traefik
# 下载traefik镜像并上传至harbor
[root@wang-200 ~]# docker pull traefik:v1.7.2-alpine
[root@wang-200 ~]# docker tag traefik:v1.7.2-alpine harbor.wang.com/public/traefik:v1.7.2-alpine[root@wang-200 ~]# docker push harbor.wang.com/public/traefik:v1.7.2-alpine
18-2、准备资源配置清单
温馨提示：wang-200节点执行
18-2-1、rbac资源配置清单
[root@wang-200 ~]# mkdir /data/k8s-yaml/traefik
[root@wang-200 ~]# vim /data/k8s-yaml/traefik/rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: traefik-ingress-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: traefik-ingress-controller
rules:
- apiGroups:
- ""
resources:
- services
- endpoints
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- ingresses
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
name: traefik-ingress-controller
namespace: kube-system
18-2-2、ds资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/traefik/ds.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: traefik-ingress
namespace: kube-system
labels:
k8s-app: traefik-ingress
spec:
template:
metadata:
labels:
k8s-app: traefik-ingress
name: traefik-ingress
spec:
serviceAccountName: traefik-ingress-controller
terminationGracePeriodSeconds: 60
containers:
- image: harbor.wang.com/public/traefik:v1.7.2-alpine
name: traefik-ingress
ports:
- name: controller
containerPort: 80
hostPort: 81
- name: admin-web
containerPort: 8080
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
args:
- --api
- --kubernetes
- --logLevel=INFO
- --insecureskipverify=true
- --kubernetes.endpoint=https://10.0.0.99:7443
- --accesslog
- --accesslog.filepath=/var/log/traefik_access.log
- --traefiklog
- --traefiklog.filepath=/var/log/traefik.log
- --metrics.prometheus
18-2-3、svc资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/traefik/svc.yaml
[root@wang-200 ~]# cat /data/k8s-yaml/traefik/svc.yaml
kind: Service
apiVersion: v1
metadata:
name: traefik-ingress-service
namespace: kube-system
spec:
selector:
k8s-app: traefik-ingress
ports:
- protocol: TCP
port: 80
name: controller
- protocol: TCP
port: 8080
name: admin-web
18-2-4、ingress资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/traefik/ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: traefik-web-ui
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: traefik.wang.com
http:
paths:
- path: /
backend:
serviceName: traefik-ingress-service
servicePort: 8080
18-3、创建资源
温馨提示：wang-47或wang-37其中一个节点执行
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/traefik/rbac.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/traefik/ds.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/traefik/svc.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/traefik/ingress.yaml
# 按照配置已经监听了宿主机的81端口
[root@wang-47 ~]# ss -ntlp | grep 81
LISTEN 0 128 *:81 *:* users:(("docker-proxy",pid=81576,fd=4))
LISTEN 0 128 [::]:81 [::]:* users:(("docker-proxy",pid=81580,fd=4))
# 注意：使用kubectl get pod -n kube-system,traefik这个pod可能会停留在containercreating的状态，并报错
Warning FailedCreatePodSandBox 15m kubelet, kjdow7-21.host.com Failed create pod sandbox: rpc error: code = Unknown desc = failed to start sandbox container for pod "traefik-ingress-7fsp8": Error response from daemon: driver failed programming external connectivity on endpoint k8s_POD_traefik-ingress-7fsp8_kube-system_3c7fbecb-801c-4f3a-aa30-e3717245d9f5_0 (7603ab3cbc43915876ab0db527195a963f8c8f4a59f8a1a84f00332f3f387227): (iptables failed: iptables --wait -t filter -A DOCKER ! -i docker0 -o docker0 -p tcp -d 172.7.21.5 --dport 80 -j ACCEPT: iptables: No chain/target/match by that name.
(exit status 1))
这时可以重启kubelet服务
18-4、配置反向代理
温馨提示：wang-17、wang-27节点执行
[root@wang-17 ~]# vim /etc/nginx/conf.d/traefik.wang.com.conf
upstream default_traefik {
server 10.0.0.37:81 max_fails=3 fail_timeout=10s;
server 10.0.0.47:81 max_fails=3 fail_timeout=10s;
}
server {
server_name traefik.wang.com;
location / {
proxy_pass http://default_traefik;
proxy_set_header Host $http_host;
proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
}
}
[root@wang-17 ~]# nginx -t
[root@wang-17 ~]# nginx -s reload
18-5、配置dns
温馨提示：wang-17节点执行
[root@wang-17 ~]# vim /var/named/wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112504 ; serial
108000 ; refresh
900 ; retry
604800 ; expire
86400 ) ; minimum
NS dns.wang.com.
dns A 10.0.0.17
harbor A 10.0.0.200
k8s-yaml A 10.0.0.200
traefik A 10.0.0.99
[root@wang-17 ~]# systemctl restart named
[root@wang-17 ~]# systemctl status named
[root@wang-17 ~]# dig -t A traefik.wang.com @10.0.0.17 +short
10.0.0.99
浏览器访问：traefik.wang.com
19、部署dashboard
GitHub下载地址：https://github.com/kubernetes/dashboard
Docker下载地址：https://hub.docker.com/r/k8scn/kubernetes-dashboard-amd64
19-1、下载dashboard
温馨提示：wang-200节点执行
# 下载dashboard镜像并上传至harbor
[root@wang-200 ~]# docker pull k8scn/kubernetes-dashboard-amd64:v1.8.3
[root@wang-200 ~]# docker tag k8scn/kubernetes-dashboard-amd64:v1.8.3
[root@wang-200 ~]# docker push harbor.wang.com/public/kubernetes-dashboard-amd64:v1.8.3
19-2、准备资源配置清单
温馨提示：wang-200节点执行
19-2-1、rbac资源配置清单
[root@wang-200 ~]# mkdir /data/k8s-yaml/dashboard
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard-admin
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard-admin
namespace: kube-system
19-2-2、dp资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/dp.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
priorityClassName: system-cluster-critical
containers:
- name: kubernetes-dashboard
image: harbor.wang.com/public/kubernetes-dashboard-amd64:v1.8.3
resources:
limits:
cpu: 100m
memory: 300Mi
requests:
cpu: 50m
memory: 100Mi
ports:
- containerPort: 8443
protocol: TCP
args:
# PLATFORM-SPECIFIC ARGS HERE
- --auto-generate-certificates
volumeMounts:
- name: tmp-volume
mountPath: /tmp
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard-admin
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
19-2-3、svc资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/svc.yaml
apiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
k8s-app: kubernetes-dashboard
ports:
- port: 443
targetPort: 8443
19-2-4、ingress资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: dashboard.wang.com
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 443
19-3、创建资源
温馨提示：wang-47或wang-37其中一个节点执行
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/rbac.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/dp.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/svc.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/ingress.yaml
[root@wang-47 ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-567ff85894-jvqrg 1/1 Running 0 76m
kubernetes-dashboard-7b4759bccf-ncv9k 1/1 Running 0 50s
traefik-ingress-kj998 1/1 Running 0 38m
traefik-ingress-sskvw 1/1 Running 0 38m
[root@wang-47 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
coredns ClusterIP 192.168.0.2 <none> 53/UDP,53/TCP,9153/TCP 76m
kubernetes-dashboard ClusterIP 192.168.29.89 <none> 443/TCP 60s
traefik-ingress-service ClusterIP 192.168.81.80 <none> 80/TCP,8080/TCP 38m
[root@wang-47 ~]# kubectl get ingress -n kube-system
NAME HOSTS ADDRESS PORTS AGE
kubernetes-dashboard dashboard.wang.com 80 108s
traefik-web-ui traefik.wang.com 80 39m
19-4、配置dns
温馨提示：wang-17节点执行
[root@wang-17 ~]# vim /var/named/wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112505 ; serial
108000 ; refresh
900 ; retry
604800 ; expire
86400 ) ; minimum
NS dns.wang.com.
dns A 10.0.0.17
harbor A 10.0.0.200
k8s-yaml A 10.0.0.200
traefik A 10.0.0.99
dashboard A 10.0.0.99
[root@wang-17 ~]# systemctl restart named
[root@wang-17 ~]# dig -t A dashboard.wang.com @10.0.0.17 +short
10.0.0.99
# 在k8s集群验证解析
[root@wang-47 ~]# dig -t A dashboard.wang.com @192.168.0.2 +short
10.0.0.99
19-5、登录dashboard
温馨提示：wang-17节点执行
[root@wang-17 ~]# mkdir /etc/nginx/certs
19-5-1、配置访问dashboard的https证书
温馨提示：wang-200节点执行
[root@wang-200 ~]# cd /opt/certs/
[root@wang-200 certs]# (umask 077;openssl genrsa -out dashboard.wang.com.key 2048)
[root@wang-200 certs]# openssl req -new -key dashboard.wang.com.key -out dashboard.wang.com.csr -subj "/CN=dashboard.wang.com/C=CN/ST=BJ/L=beijing/O=wang/OU=WANG"
[root@wang-200 certs]# openssl x509 -req -in dashboard.wang.com.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out dashboard.wang.com.crt -days 3650
[root@wang-200 certs]# scp dashboard.wang.com.crt dashboard.wang.com.key 10.0.0.17:/etc/nginx/certs/
dashboard.wang.com.crt 100% 1188 1.1MB/s 00:00
dashboard.wang.com.key
19-5-2、配置nginx代理dashboard使用https
温馨提示：wang-17节点执行
[root@wang-17 ~]# vim /etc/nginx/conf.d/dashboard.wang.com.conf
server {
listen 80;
server_name dashboard.wang.com;
rewrite ^(.*)$ https://${server_name}$1 permanent;
}
server {
listen 443 ssl;
server_name dashboard.wang.com;
ssl_certificate "certs/dashboard.wang.com.crt";
ssl_certificate_key "certs/dashboard.wang.com.key";
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 10m;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
location / {
proxy_pass http://default_traefik;
proxy_set_header Host $http_host;
proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
}
}
[root@wang-17 ~]# nginx -t
[root@wang-17 ~]# nginx -s reload
19-5-3、查看token
温馨提示：wang-47节点执行
[root@wang-47 ~]# kubectl get secret -n kube-system
NAME TYPE DATA AGE
coredns-token-tnb6p kubernetes.io/service-account-token 3 109m
default-token-4bc26 kubernetes.io/service-account-token 3 7h49m
kubernetes-dashboard-admin-token-z5pk4 kubernetes.io/service-account-token 3 33m
kubernetes-dashboard-key-holder Opaque 2 33m
traefik-ingress-controller-token-sfnxr kubernetes.io/service-account-token 3 71m
[root@wang-47 ~]# kubectl describe secret kubernetes-dashboard-admin-token-z5pk4 -n kube-system
Name: kubernetes-dashboard-admin-token-z5pk4
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: kubernetes-dashboard-admin
kubernetes.io/service-account.uid: cd6b9f19-acb0-4716-82a6-c5643221aa3f
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1285 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbi10b2tlbi16NXBrNCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImNkNmI5ZjE5LWFjYjAtNDcxNi04MmE2LWM1NjQzMjIxYWEzZiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTprdWJlcm5ldGVzLWRhc2hib2FyZC1hZG1pbiJ9.HqX3RbJ-Bs1ItaHGmgyYOL6MRcwUfSDOnxOceEGtAd_7wJ4SLdyWDnnt8LoEVVc8rkwDVnP7pY5DpyyPIABhA3P02aMX2EOX_3blshrwr0RJvJXfxIEN2nknOA5DKbi5EKmNqnphAf8eEagY99mduCpk-APHRjStCUdOjmK3nSKDUK8e4LPNWbpVHk19EoK3HWAuAnavlmvSo_lCl5JX-iCDAEh1U6ETJk9FxYZCTrboOOOaMyQiDFvBzhhCmajOoGNyd7fuzz_7vDBP7xSN0QBgVjIagCby3tFv6YbFVL-6jVOtLDPjMH4Wr0iz7qrLkoD5StVXPVbDfpa5Vgg1-g
复制最后token字段的值，并粘贴
19-5-4、升级dashboard版本至1.10.1
温馨提示：wang-200节点执行
[root@wang-200 ~]# docker pull rancher/kubernetes-dashboard-amd64:v1.10.1
[root@wang-200 ~]# docker tag rancher/kubernetes-dashboard-amd64:v1.10.1 harbor.wang.com/public/kubernetes-dashboard-amd64:v1.10.1
[root@wang-200 ~]# sed -i 's@dashboard-amd64:v1.8.3@dashboard-amd64:v1.10.1@g' /opt/apps/harbor/dp.yaml
温馨提示：wang-47节点执行
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/dp.yaml
[root@wang-47 ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-567ff85894-jvqrg 1/1 Running 0 18h
kubernetes-dashboard-746cfffbfd-gwvp5 1/1 Running 0 8s
kubernetes-dashboard-7b4759bccf-ncv9k 0/1 Terminating 0 17h
traefik-ingress-kj998 1/1 Running 0 18h
traefik-ingress-sskvw 1/1 Running 0 18h
[root@wang-47 ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-567ff85894-jvqrg 1/1 Running 0 18h
kubernetes-dashboard-746cfffbfd-gwvp5 1/1 Running 0 11s
traefik-ingress-kj998 1/1 Running 0 18h
traefik-ingress-sskvw 1/1 Running 0 18h
19-6、安装dashboard插件heapster
温馨提示：wang-200节点执行
19-6-1、下载heapster镜像
# 下载heapster镜像并将镜像上传至harbor
[root@wang-200 ~]# docker pull bitnami/heapster:1.5.4
[root@wang-200 ~]# docker tag bitnami/heapster:1.5.4 harbor.wang.com/public/heapster:1.5.4
[root@wang-200 ~]# docker push harbor.wang.com/public/heapster:1.5.4
19-6-2、rbac资源配置清单
[root@wang-200 ~]# mkdir /data/k8s-yaml/dashboard/heapster
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/heapster/rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: heapster
namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: heapster
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:heapster
subjects:
- kind: ServiceAccount
name: heapster
namespace: kube-system
19-6-2、dp资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/heapster/dp.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: heapster
namespace: kube-system
spec:
replicas: 1
template:
metadata:
labels:
task: monitoring
k8s-app: heapster
spec:
serviceAccountName: heapster
containers:
- name: heapster
image: harbor.wang.com/public/heapster:1.5.4
imagePullPolicy: IfNotPresent
command:
- /opt/bitnami/heapster/bin/heapster
- --source=kubernetes:https://kubernetes.default
19-6-3、svc资源配置清单
[root@wang-200 ~]# vim /data/k8s-yaml/dashboard/heapster/svc.yaml
apiVersion: v1
kind: Service
metadata:
labels:
task: monitoring
# For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
# If you are NOT using this as an addon, you should comment out this line.
kubernetes.io/cluster-service: 'true'
kubernetes.io/name: Heapster
name: heapster
namespace: kube-system
spec:
ports:
- port: 80
targetPort: 8082
selector:
k8s-app: heapster
19-6-4、应用资源配置清单
温馨提示：wang-47节点执行
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/heapster/rbac.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/heapster/dp.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/dashboard/heapster/svc.yaml
[root@wang-47 ~]# kubectl get pod -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-567ff85894-jvqrg 1/1 Running 0 19h
heapster-6c8db4b86b-rp5ml 1/1 Running 0 85s
kubernetes-dashboard-746cfffbfd-gwvp5 1/1 Running 0 32m
traefik-ingress-kj998 1/1 Running 0 18h
traefik-ingress-sskvw 1/1 Running 0 18h
20、部署zookeeper
温馨提示：wang-17、wang-27、wang-37节点执行
20-1、安装jdk
[root@wang-17 ~]# mkdir /usr/java
[root@wang-17 ~]# tar xf jdk-8u341-linux-x64.tar.gz -C /usr/java/
[root@wang-17 ~]# ln -s /usr/java/jdk1.8.0_341/ /usr/java/jdk
[root@wang-17 ~]# vim /etc/profile
export JAVA_HOME=/usr/java/jdk
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar
[root@wang-17 ~]# source /etc/profile
[root@wang-17 ~]# java -version
java version "1.8.0_341"
Java(TM) SE Runtime Environment (build 1.8.0_341-b10)
Java HotSpot(TM) 64-Bit Server VM (build 25.341-b10, mixed mode)
20-2、安装zookeeper
官方下载地址：https://archive.apache.org/dist/zookeeper/
[root@wang-17 ~]# tar xf zookeeper-3.4.14.tar.gz -C /opt
[root@wang-17 ~]# ln -s /opt/zookeeper-3.4.14/ /opt/zookeeper
[root@wang-17 ~]# mkdir -p /data/zookeeper/data /data/zookeeper/logs
[root@wang-17 ~]# cp /opt/zookeeper/conf/zoo_sample.cfg /opt/zookeeper/conf/zoo.cfg
# 各节点zk配置相同
[root@wang-17 ~]# vim /opt/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
clientPort=2181
server.1=zk1.wang.com:2888:3888
server.2=zk2.wang.com:2888:3888
server.3=zk3.wang.com:2888:3888
# 配置id，分别在三个节点执行：
# 注意：各节点id不一样
[root@wang-17 ~]# cat /data/zookeeper/data/myid
1
[root@wang-27 ~]# cat /data/zookeeper/data/myid
2
[root@wang-37 ~]# cat /data/zookeeper/data/myid
3
20-3、配置dns解析
温馨提示：wang-17节点执行
[root@wang-17 ~]# vim /var/named/wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112506 ; serial
108000 ; refresh
900 ; retry
604800 ; expire
86400 ) ; minimum
NS dns.wang.com.
dns A 10.0.0.17
harbor A 10.0.0.200
k8s-yaml A 10.0.0.200
traefik A 10.0.0.99
dashboard A 10.0.0.99
zk1 A 10.0.0.17
zk2 A 10.0.0.27
zk3 A 10.0.0.37
[root@wang-17 ~]# systemctl restart named
[root@wang-17 ~]# dig -t A zk1.wang.com @10.0.0.17 +short
10.0.0.17
[root@wang-17 ~]# dig -t A zk3.wang.com @10.0.0.17 +short
10.0.0.37
20-4、启动zookeeper
# 各节点依次启动
[root@wang-17 ~]# /opt/zookeeper/bin/zkServer.sh start
[root@wang-17 ~]# /opt/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper/bin/../conf/zoo.cfg
Mode: follower
[root@wang-27 ~]# /opt/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper/bin/../conf/zoo.cfg
Mode: leader
[root@wang-37 ~]# /opt/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper/bin/../conf/zoo.cfg
Mode: follower
21、部署jenkins
温馨提示：wang-200节点执行
21-1、下载镜像
官方下载地址：https://hub.docker.com/r/jenkins/jenkins
# 下载jenkins镜像并上传至harbor
[root@wang-200 ~]# docker pull jenkins/jenkins:2.190.3
[root@wang-200 ~]# docker tag jenkins/jenkins:2.190.3 harbor.wang.com/public/jenkins:2.190.3
[root@wang-200 ~]# docker push harbor.wang.com/public/jenkins:2.190.3
21-2、自定义Dockerfile
官网拉取的镜像需要做些自定义操作，才能在k8s集群中部署
[root@wang-200 ~]# mkdir -p dat/dockerfile/jenkins
[root@wang-200 ~]# cd dat/dockerfile/jenkins
[root@wang-200 jenkins]# vim Dockerfile
FROM harbor.wang.com/public/jenkins:2.190.3
USER root
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
echo 'Asia/Shanghai' >/etc/timezone
ADD id_rsa /root/.ssh/id_rsa
ADD config.json /root/.docker/config.json
ADD get-docker.sh /get-docker.sh
RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config &&\
/get-docker.sh
# 说明：
这个Dockerfile里我们主要做了以下几件事：
设置容器用户为root
设置容器内的时区
将ssh私钥加入（使用git拉代码时要用到，配对的公钥应配置在gitlab中）
加入了登录自建harbor仓库的config文件
修改了ssh客户端的
安装一个docker的客户端
如果因为网络原因构建失败，可以在最后“/get-docker.sh --mirror Aliyun”
21-2-1、生成ssk密钥对
[root@wang-200 jenkins]# ssh-keygen -t rsa -b 2048 -C "965507991@qq.com" -N "" -f /root/.ssh/id_rsa
Generating public/private rsa key pair.
/root/.ssh/id_rsa already exists.
Overwrite (y/n)? y
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:qS3flDgSFTUg1HViaq7wcggCbGmdnu1HgKVTyS1Zn/0 965507991@qq.com
The key's randomart image is:
+---[RSA 2048]----+
| .+Boo+* . |
|. o *= oo+o+ |
|.= * .. +o . |
|+ . + .+ . . |
|. .o....S E |
| . ..+.= . . |
| o.B.+ o |
| o.+ + |
| . . |
+----[SHA256]-----+
21-2-2、准备其他文件
[root@wang-200 jenkins]# cp /root/.ssh/id_rsa .
[root@wang-200 jenkins]# cp /root/.docker/config.json .
[root@wang-200 jenkins]# curl -fsSL get.docker.com -o get-docker.sh
[root@wang-200 jenkins]# chmod +x get-docker.sh
[root@wang-200 jenkins]# ll
total 32
-rw------- 1 root root 83 Nov 27 18:48 config.json
-rw-r--r-- 1 root root 348 Nov 27 18:45 Dockerfile
-rwxr-xr-x 1 root root 19873 Nov 27 18:48 get-docker.sh
-rw------- 1 root root 1675 Nov 27 18:49 id_rsa
21-2-3、登录harbor仓库页面,创建infra
创建infra的project，access level 为Private
21-2-4、生成镜像
[root@wang-200 jenkins]# docker build -t harbor.wang.com/infra/jenkins:2.190.3 .
[root@wang-200 jenkins]# docker push harbor.wang.com/infra/jenkins:2.190.3
21-3、准备共享存储
21-3-1、安装nfs
温馨提示：所有node节点和wang-200节点执行
[root@wang-200 jenkins]# yum install -y nfs-utils
21-3-2、配置nfs服务
温馨提示：wang-200节点执行
[root@wang-200 jenkins]# mkdir -p /data/nfs-volume
[root@wang-200 jenkins]# vim /etc/exports
/data/nfs-volume 10.0.0.0/24(rw,no_root_squash)
[root@wang-200 jenkins]# systemctl start nfs
[root@wang-200 jenkins]# systemctl enable nfs
[root@wang-200 jenkins]# systemctl status nfs
[root@wang-200 jenkins]# showmount -e 10.0.0.200
Export list for 10.0.0.200:
21-4、准备资源配置清单
温馨提示：wang-200节点执行
21-4-1、docker-registry密钥资源清单
[root@wang-200 jenkins]# mkdir /data/k8s-yaml/jenkins
[root@wang-200 jenkins]# mkdir /data/nfs-volume/jenkins_home
[root@wang-200 jenkins]# cd /data/k8s-yaml/jenkins/
# 说明：docker-registory 登陆密钥
# 在wang-47节点使用 kubectl create secret docker-registry docker-secret --docker-server=harbor.wang.com --docker-username=admin --docker-password=harbor@12345 --dry-run -o yaml
生成的，然后拷贝过来
[root@wang-200 jenkins]# vim harbor-secret.yaml
apiVersion: v1
data:
.dockerconfigjson: eyJhdXRocyI6eyJoYXJib3Iud2FuZy5jb20iOnsidXNlcm5hbWUiOiJhZG1pbiIsInBhc3N3b3JkIjoiaGFyYm9yQDEyMzQ1IiwiYXV0aCI6IllXUnRhVzQ2YUdGeVltOXlRREV5TXpRMSJ9fX0=
kind: Secret
metadata:
creationTimestamp: null
name: harbor-secret
namespace: infra
type: kubernetes.io/dockerconfigjson
21-4-2、dp资源配置清单
[root@wang-200 jenkins]# vim dp.yaml
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
name: jenkins
namespace: infra
labels:
name: jenkins
spec:
replicas: 1
selector:
matchLabels:
name: jenkins
template:
metadata:
labels:
app: jenkins
name: jenkins
spec:
volumes:
- name: data
nfs:
server: wang-200
path: /data/nfs-volume/jenkins_home
- name: docker
hostPath:
path: /run/docker.sock
type: ''
containers:
- name: jenkins
image: harbor.wang.com/infra/jenkins:2.190.3
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
protocol: TCP
env:
- name: JAVA_OPTS
value: -Xmx512m -Xms512m
volumeMounts:
- name: data
mountPath: /var/jenkins_home
- name: docker
mountPath: /run/docker.sock
imagePullSecrets:
- name: harbor-secret
securityContext:
runAsUser: 0
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
revisionHistoryLimit: 7
progressDeadlineSeconds: 600
# 注：imagePullSecrets:中name在创建secret时指定过的
将宿主机/run/docker.sock挂载到pod中，那么pod就可以与宿主机的docker的server端进行通信了
21-4-3、svc资源配置清单
[root@wang-200 jenkins]# vim svc.yaml
kind: Service
apiVersion: v1
metadata:
name: jenkins
namespace: infra
spec:
ports:
- protocol: TCP
port: 80
targetPort: 8080
selector:
app: jenkins
# 注： targetport指定镜像端口，jenkins默认打开页面端口是8080
port指定暴露给service中的cluster ip使用80端口
21-4-4、ingress资源配置清单
[root@wang-200 jenkins]# vim ingress.yaml
[root@wang-200 jenkins]# cat ingress.yaml
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: jenkins
namespace: infra
spec:
rules:
- host: jenkins.wang.com
http:
paths:
- path: /
backend:
serviceName: jenkins
servicePort: 80
21-5、应用资源配置清单
温馨提示：wang-47节点执行
[root@wang-47 ~]# kubectl create ns infra
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/jenkins/harbor-secret.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/jenkins/dp.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/jenkins/svc.yaml
[root@wang-47 ~]# kubectl apply -f http://k8s-yaml.wang.com/jenkins/ingress.yaml
21-6、配置dns解析
温馨提示：wang-17节点执行
[root@wang-17 ~]# vim /var/named/wang.com.zone
$TTL 1D
@ IN SOA dns.wang.com. dnsadmin.wang.com. (
2022112507 ; serial
108000 ; refresh
900 ; retry
604800 ; expire
86400 ) ; minimum
NS dns.wang.com.
dns A 10.0.0.17
harbor A 10.0.0.200
k8s-yaml A 10.0.0.200
traefik A 10.0.0.99
dashboard A 10.0.0.99
zk1 A 10.0.0.17
zk2 A 10.0.0.27
zk3 A 10.0.0.37
jenkins A 10.0.0.99
[root@wang-17 ~]# systemctl restart named
[root@wang-17 ~]# dig -t A jenkins.wang.com @10.0.0.17 +short
10.0.0.99
21-7、查看jenkins初始密码
温馨提示：wang-200节点执行
# 方法一：
wang-200节点执行：
[root@wang-200 jenkins]# cat /data/nfs-volume/jenkins_home/secrets/initialAdminPassword
51dd9fc0fd3a48ecbe55122a24c30191
# 方法二：
任一master节点执行：
[root@wang-47 ~]# kubectl get pod -n infra
[root@wang-47 ~]# kubectl log -f jenkins-594888cd64-vm7jw -n infra
Jenkins initial setup is required. An admin user has been created and a password generated.
Please use the following password to proceed to installation:
51dd9fc0fd3a48ecbe55122a24c30191
This may also be found at: /var/jenkins_home/secrets/initialAdminPassword
21-8、登录jenkins并配置
22、安装部署maven
温馨提示：wang-200节点执行
官网下载地址：https://archive.apache.org/dist/maven/maven-3/
[root@wang-200 ~]# wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
[root@wang-200 ~]# tar xf apache-maven-3.6.3-bin.tar.gz -C /data/nfs-volume/jenkins_home/
[root@wang-200 ~]# cd /data/nfs-volume/jenkins_home/
[root@wang-200 jenkins_home]# mv apache-maven-3.6.3/ maven-3.6.2-8u232
[root@wang-200 jenkins_home]# vim /data/nfs-volume/jenkins_home/maven-3.6.2-8u232/conf/settings.xml
<mirrors>
<mirror>
<id>alimaven</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
<mirrorOf>central</mirrorOf>
</mirror>
</mirrors>
[root@wang-200 ~]# docker tag stanleyws/jre8:8u112 harbor.wang.com/public/jre8:8u112
[root@wang-200 ~]# docker push harbor.wang.com/public/jre8:8u112
[root@wang-200 ~]# mkdir /data/dockerfile/jre8 -p
[root@wang-200 ~]# cd /data/dockerfile/jre8/
[root@wang-200 jre8]# vim Dockerfile
FROM docker.io/stanleyws/jre8:8u112
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
echo 'Asia/Shanghai' >/etc/timezone
ADD config.yaml /opt/prom/config.yaml
ADD jmx_javaagent-0.3.1.jar /opt/prom/
WORKDIR /opt/project_dir
ADD entrypoint.sh /entrypoint.sh
CMD ["/entrypoint.sh"]
harbor页面创建object 在harbor中创建base的object，用来存放所有业务基础镜像.权限为公开
[root@wang-200 jre8]# wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar -O jmx_javaagent-0.3.1.jar
[root@wang-200 jre8]# vim config.ymal
---
rules:
- pattern: '.*'
[root@wang-200 jre8]# vim entrypoint.sh
#!/bin/sh
M_OPTS="-Duser.timezone=Asia/Shanghai -javaagent:/opt/prom/jmx_javaagent-0.3.1.jar=$(hostname -i):${M_PORT:-"12346"}:/opt/prom/config.yaml"
C_OPTS=${C_OPTS}
JAR_BALL=${JAR_BALL}
exec java -jar ${M_OPTS} ${C_OPTS} ${JAR_BALL}
[root@wang-200 jre8]# chmod +x entrypoint.sh
[root@wang-200 jre8]# docker build -t harbor.wang.com/base/jre8:8u112 .