kubernetes v1.18.2 二进制部署 ipv4 etcd 部署

1、系统相关设置

修改服务器名

ssh 192.168.2.175 hostnamectl set-hostname k8s-master-1  
ssh 192.168.2.176 hostnamectl set-hostname k8s-master-2  
ssh 192.168.2.177 hostnamectl set-hostname k8s-master-3  
ssh 192.168.2.185 hostnamectl set-hostname k8s-node-1    
ssh 192.168.2.187 hostnamectl set-hostname k8s-node-2

设置关闭防火墙及SELINUX

# centosx
sed -i ‘s/SELINUX=.*/SELINUX=disabled/g‘ /etc/selinux/config
systemctl stop firewalld && systemctl disable firewalld
setenforce 0
# Ubuntu
systemctl stop ufw.service
systemctl disable ufw.service

安装及配置CFSSL 签发证书使用

#go 环境部署
yum install go
vi ~/.bash_profile
GOBIN=/root/go/bin/
PATH=$PATH:$GOBIN:$HOME/bin
export PATH
go get  github.com/cloudflare/cfssl/cmd/cfssl
go get  github.com/cloudflare/cfssl/cmd/cfssljson

签发etcd 证书

# 设置证书环境变量
# 设置证书使用时间87600h 10年
export EXPIRY_TIME="87600h"
# 签发证书IP
export ETCD_MEMBER_1_IP="192.168.2.175"
export ETCD_MEMBER_2_IP="192.168.2.176"
export ETCD_MEMBER_3_IP="192.168.2.177"
# 机器名
export ETCD_MEMBER_1_HOSTNAMES="k8s-master-1"
export ETCD_MEMBER_2_HOSTNAMES="k8s-master-2"
export ETCD_MEMBER_3_HOSTNAMES="k8s-master-3"
# etcd 集群通讯证书
export ETCD_SERVER_HOSTNAMES="\"${ETCD_MEMBER_1_HOSTNAMES}\",\"${ETCD_MEMBER_2_HOSTNAMES}\",\"${ETCD_MEMBER_3_HOSTNAMES}\""
export ETCD_SERVER_IPS="\"${ETCD_MEMBER_1_IP}\",\"${ETCD_MEMBER_2_IP}\",\"${ETCD_MEMBER_3_IP}\""
#证书所需要的配置参数
export CERT_ST="GuangDong"
export CERT_L="GuangZhou"
export CERT_O="k8s"
export CERT_OU="Qist"
export CERT_PROFILE="kubernetes"
# 设置工作目录
export  HOST_PATH=`pwd`
# 创建etcd K8S 证书json 存放目录
mkdir -p ${HOST_PATH}/cfssl/{k8s,etcd}
# 创建签发证书存放目录
mkdir -p ${HOST_PATH}/cfssl/pki/{k8s,etcd}
# CA 配置文件用于配置根证书的使用场景 (profile) 和具体参数 (usage,过期时间、服务端认证、客户端认证、加密等),后续在签名其它证书时需要指定特定场景。
cat << EOF | tee ${HOST_PATH}/cfssl/ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${EXPIRY_TIME}"
    },
    "profiles": {
      "${CERT_PROFILE}": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "${EXPIRY_TIME}"
      }
    }
  }
}
EOF
# 创建 ETCD CA 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json
{
    "CN": "etcd",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ],
    "ca": {
        "expiry": "${EXPIRY_TIME}"
    }
}
EOF
# etcd ca 证书签发
cfssl gencert -initca ${HOST_PATH}/cfssl/etcd/etcd-ca-csr.json |       cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-ca
# 创建 ETCD Server 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-server.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    ${ETCD_SERVER_IPS},
    ${ETCD_SERVER_HOSTNAMES}
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 ETCD Server 证书和私钥
cfssl gencert     -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem     -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem     -config=${HOST_PATH}/cfssl/ca-config.json     -profile=${CERT_PROFILE}     ${HOST_PATH}/cfssl/etcd/etcd-server.json |     cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-server
# 创建 ETCD Member 1 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${ETCD_MEMBER_1_IP}",
    "${ETCD_MEMBER_1_HOSTNAMES}"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 ETCD Member 1 证书和私钥
cfssl gencert     -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem     -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem     -config=${HOST_PATH}/cfssl/ca-config.json     -profile=${CERT_PROFILE}     ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json |     cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_1_HOSTNAMES}
# 创建 ETCD Member 2 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${ETCD_MEMBER_2_IP}",
    "${ETCD_MEMBER_2_HOSTNAMES}"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 ETCD Member 2 证书和私钥
cfssl gencert     -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem     -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem     -config=${HOST_PATH}/cfssl/ca-config.json     -profile=${CERT_PROFILE}     ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_2_HOSTNAMES}.json |     cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_2_HOSTNAMES}
# 创建 ETCD Member 3 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "${ETCD_MEMBER_3_IP}",
    "${ETCD_MEMBER_3_HOSTNAMES}"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 ETCD Member 3 证书和私钥
cfssl gencert     -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem     -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem     -config=${HOST_PATH}/cfssl/ca-config.json     -profile=${CERT_PROFILE}     ${HOST_PATH}/cfssl/etcd/${ETCD_MEMBER_3_HOSTNAMES}.json |     cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-member-${ETCD_MEMBER_3_HOSTNAMES}
# 创建 ETCD Client 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/etcd/etcd-client.json
{
  "CN": "client",
  "hosts": [""], 
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
            "O": "$CERT_O",
            "OU": "$CERT_OU"
    }
  ]
}
EOF
# 生成 ETCD Client 证书和私钥
cfssl gencert     -ca=${HOST_PATH}/cfssl/pki/etcd/etcd-ca.pem     -ca-key=${HOST_PATH}/cfssl/pki/etcd/etcd-ca-key.pem     -config=${HOST_PATH}/cfssl/ca-config.json     -profile=${CERT_PROFILE}     ${HOST_PATH}/cfssl/etcd/etcd-client.json |     cfssljson -bare ${HOST_PATH}/cfssl/pki/etcd/etcd-client
# 分发生成的证书到所有需要部署etcd 节点
ssh 192.168.2.175 mkdir -p /apps/etcd/ssl
ssh 192.168.2.176 mkdir -p /apps/etcd/ssl
ssh 192.168.2.177 mkdir -p /apps/etcd/ssl
# 分发文件
scp -r ./cfssl/pki/etcd/* 192.168.2.175:/apps/etcd/ssl/
scp -r ./cfssl/pki/etcd/* 192.168.2.176:/apps/etcd/ssl/
scp -r ./cfssl/pki/etcd/* 192.168.2.177:/apps/etcd/ssl/

2、etcd 二进制文件准备

wget https://github.com/etcd-io/etcd/releases/download/v3.4.7/etcd-v3.4.7-linux-amd64.tar.gz
# 解压下载好文件
tar -xvf etcd-v3.4.7-linux-amd64.tar.gz
# 创建二进制远程存放目录
ssh 192.168.2.175 mkdir -p /apps/etcd/bin
ssh 192.168.2.176 mkdir -p /apps/etcd/bin
ssh 192.168.2.177 mkdir -p /apps/etcd/bin
# 分发解压好二进制文件
cd etcd-v3.4.7-linux-amd64/
scp -r etcd* 192.168.2.175:/apps/etcd/bin
scp -r etcd* 192.168.2.176:/apps/etcd/bin
scp -r etcd* 192.168.2.177:/apps/etcd/bin

3、etcd 配置文件准备

# 创建配置文件存放目录
ssh 192.168.2.175 mkdir -p /apps/etcd/conf
ssh 192.168.2.176 mkdir -p /apps/etcd/conf
ssh 192.168.2.177 mkdir -p /apps/etcd/conf
# 192.168.2.175   配置
ssh 192.168.2.175 
cat << EOF | tee /apps/etcd/conf/etcd
ETCD_OPTS="--name=k8s-master-1 \           --data-dir=/apps/etcd/data/default.etcd \           --wal-dir=/apps/etcd/data/default.etcd/wal \           --listen-peer-urls=https://192.168.2.175:2380 \           --listen-client-urls=https://192.168.2.175:2379,https://127.0.0.1:2379 \           --advertise-client-urls=https://192.168.2.175:2379 \           --initial-advertise-peer-urls=https://192.168.2.175:2380 \           --initial-cluster=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \           --initial-cluster-token=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \           --initial-cluster-state=new \           --heartbeat-interval=6000 \           --election-timeout=30000 \           --snapshot-count=5000 \           --auto-compaction-retention=1 \           --max-request-bytes=33554432 \           --quota-backend-bytes=17179869184 \           --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \           --cert-file=/apps/etcd/ssl/etcd-server.pem \           --key-file=/apps/etcd/ssl/etcd-server-key.pem \           --peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-1.pem \           --peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-1-key.pem \           --peer-client-cert-auth \           --enable-v2=true \           --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
EOF
# 192.168.2.176 配置
ssh 192.168.2.176
cat << EOF | tee /apps/etcd/conf/etcd
ETCD_OPTS="--name=k8s-master-2 \           --data-dir=/apps/etcd/data/default.etcd \           --wal-dir=/apps/etcd/data/default.etcd/wal \           --listen-peer-urls=https://192.168.2.176:2380 \           --listen-client-urls=https://192.168.2.176:2379,https://127.0.0.1:2379 \           --advertise-client-urls=https://192.168.2.176:2379 \           --initial-advertise-peer-urls=https://192.168.2.176:2380 \           --initial-cluster=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \           --initial-cluster-token=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \           --initial-cluster-state=new \           --heartbeat-interval=6000 \           --election-timeout=30000 \           --snapshot-count=5000 \           --auto-compaction-retention=1 \           --max-request-bytes=33554432 \           --quota-backend-bytes=17179869184 \           --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \           --cert-file=/apps/etcd/ssl/etcd-server.pem \           --key-file=/apps/etcd/ssl/etcd-server-key.pem \           --peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-2.pem \           --peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-2-key.pem \           --peer-client-cert-auth \           --enable-v2=true \           --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
EOF
# 192.168.2.177 配置
ssh 192.168.2.177
cat << EOF | tee /apps/etcd/conf/etcd
ETCD_OPTS="--name=k8s-master-3 \           --data-dir=/apps/etcd/data/default.etcd \           --wal-dir=/apps/etcd/data/default.etcd/wal \           --listen-peer-urls=https://192.168.2.177:2380 \           --listen-client-urls=https://192.168.2.177:2379,https://127.0.0.1:2379 \           --advertise-client-urls=https://192.168.2.177:2379 \           --initial-advertise-peer-urls=https://192.168.2.177:2380 \           --initial-cluster=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \           --initial-cluster-token=k8s-master-1=https://192.168.2.175:2380,k8s-master-2=https://192.168.2.176:2380,k8s-master-3=https://192.168.2.177:2380 \           --initial-cluster-state=new \           --heartbeat-interval=6000 \           --election-timeout=30000 \           --snapshot-count=5000 \           --auto-compaction-retention=1 \           --max-request-bytes=33554432 \           --quota-backend-bytes=17179869184 \           --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \           --cert-file=/apps/etcd/ssl/etcd-server.pem \           --key-file=/apps/etcd/ssl/etcd-server-key.pem \           --peer-cert-file=/apps/etcd/ssl/etcd-member-k8s-master-3.pem \           --peer-key-file=/apps/etcd/ssl/etcd-member-k8s-master-3-key.pem \           --peer-client-cert-auth \           --enable-v2=true \           --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"
EOF

etcd 启动文件配置

cat << EOF | tee etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/etcd-io/etcd
[Service]
Type=notify
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
LimitMEMLOCK=infinity
User=etcd
Group=etcd
WorkingDirectory=/apps/etcd/data/default.etcd
EnvironmentFile=-/apps/etcd/conf/etcd
ExecStart=/apps/etcd/bin/etcd \$ETCD_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
# 上传启动文件到服务器
scp etcd.service 192.168.2.175:/usr/lib/systemd/system
scp etcd.service 192.168.2.176:/usr/lib/systemd/system
scp etcd.service 192.168.2.176:/usr/lib/systemd/system

etcd 启动准备

# 创建etcd 用户
ssh  192.168.2.175 useradd etcd -s /sbin/nologin -M
ssh  192.168.2.176 useradd etcd -s /sbin/nologin -M
ssh  192.168.2.177 useradd etcd -s /sbin/nologin -M
# 创建etcd 存储文件目录
ssh  192.168.2.175 mkdir -p /apps/etcd/data/default.etcd/wal
ssh  192.168.2.176 mkdir -p /apps/etcd/data/default.etcd/wal
ssh  192.168.2.177 mkdir -p /apps/etcd/data/default.etcd/wal
# 给/apps/etcd etcd 用户权限
ssh  192.168.2.175 chown -R etcd:etcd /apps/etcd/
ssh  192.168.2.176 chown -R etcd:etcd /apps/etcd/
ssh  192.168.2.177 chown -R etcd:etcd /apps/etcd/

etcd 启动

# 刷新service
systemctl daemon-reload
# 设置开机启动
systemctl enable etcd.service
# 启动etcd
systemctl  start etcd.service
# 查看启动状态
systemctl  status etcd.service
# 验证etcd 集群是否正常 任意节点
vi ~/.bashrc
export ETCDCTL_API=3
export ENDPOINTS=https://192.168.2.175:2379,https://192.168.2.176:2379,https://192.168.2.177:2379
alias ctl=‘/apps/etcd/bin/etcdctl   --endpoints=${ENDPOINTS}   --cacert=/apps/etcd/ssl/etcd-ca.pem --cert=/apps/etcd/ssl/etcd-client.pem --key=/apps/etcd/ssl/etcd-client-key.pem‘
# 保存
source  ~/.bashrc
# 验证集群是否正常
[ conf]# ctl  endpoint status
https://192.168.2.175:2379, 5ba4ebb731bb27f7, 3.4.7, 20 kB, false, false, 2, 11, 11,
https://192.168.2.176:2379, 2982ccbf31a4b4e, 3.4.7, 20 kB, true, false, 2, 11, 11,
https://192.168.2.177:2379, d86131c63ec195be, 3.4.7, 20 kB, false, false, 2, 11, 11,
[ conf]# ctl  endpoint hashkv
https://192.168.2.175:2379, 1084519789
https://192.168.2.176:2379, 1084519789
https://192.168.2.177:2379, 1084519789
[ conf]# ctl  endpoint health
https://192.168.2.176:2379 is healthy: successfully committed proposal: took = 19.088204ms
https://192.168.2.175:2379 is healthy: successfully committed proposal: took = 24.073788ms
https://192.168.2.177:2379 is healthy: successfully committed proposal: took = 26.414683ms
[ conf]# ctl member list
2982ccbf31a4b4e, started, k8s-master-2, https://192.168.2.176:2380, https://192.168.2.176:2379, false
5ba4ebb731bb27f7, started, k8s-master-1, https://192.168.2.175:2380, https://192.168.2.175:2379, false
d86131c63ec195be, started, k8s-master-3, https://192.168.2.177:2380, https://192.168.2.177:2379, false

相关推荐