OrochW's Blog

首页 归档 标签 关于

K8s+keeplived+haproxy+harbor安装集群

2022-07-18 10 min read # 云原生
ip 主机名 用途
192.168.234.31 master1 master1+etcd1+keeplived1+haproxy1
192.168.234.32 master2 master2+etcd2+keeplived2+haproxy2
192.168.234.33 master3 master3+etcd3+keeplived3+haproxy3
192.168.234.41 node1 node1
192.168.234.42 node2 node2
192.168.234.43 node3 node3
192.168.234.51 harbor harbor

一、安装keeplived

[root@master-etcd1-234-31 ~]# yum install -y keepalived
[root@master-etcd1-234-31 ~]# find /* -name keepalived*
/etc/sysconfig/keepalived
/etc/selinux/targeted/active/modules/100/keepalived
/etc/keepalived
/etc/keepalived/keepalived.conf
/usr/sbin/keepalived
/usr/lib/systemd/system/keepalived.service
/usr/share/doc/keepalived-1.3.5
/usr/share/doc/keepalived-1.3.5/keepalived.conf.SYNOPSIS
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.HTTP_GET.port
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.IPv6
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.SMTP_CHECK
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.SSL_GET
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.fwmark
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.inhibit
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.misc_check
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.misc_check_arg
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.quorum
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.sample
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.status_code
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.track_interface
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.virtual_server_group
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.virtualhost
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.localcheck
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.lvs_syncd
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.routes
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.rules
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.scripts
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.static_ipaddress
/usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp.sync
/usr/share/man/man5/keepalived.conf.5.gz
/usr/share/man/man8/keepalived.8.gz
/usr/libexec/keepalived

[root@master-etcd1-234-31 ~]# cp /usr/share/doc/keepalived-1.3.5/samples/keepalived.conf.vrrp /etc/keepalived/

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen
   }
   notification_email_from Alexandre.Cassen@firewall.loc
  #smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}


vrrp_instance VI_1 {
    state MASTER #234.32 234.33改为BACKUP
    interface ens34 #修改网络接口
    garp_master_delay 10
    smtp_alert
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.234.61/24 dev ens34 label ens34:0 #需要配置的vip 192.168.234.61
    }
}


VIP234.61到机器234.33上面了

二、安装haroxy

[root@master-etcd1-234-31 ~]# yum install -y haproxy
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    tcp#tcp
    log                     global
    option                  tcplog#tcplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
listen k8s-6443
        bind 192.168.234.61:6443
        mode  tcp
        server 192.168.234.31 192.168.234.31:6443 check inter 2s fall 3 rise 3
        server 192.168.234.32 192.168.234.32:6443 check inter 2s fall 3 rise 3
        server 192.168.234.33 192.168.234.33:6443 check inter 2s fall 3 rise 3


已经监听了vip 192.168.234.61:6443端口

监听不存在的地址和端口需要修改内核参数
echo "net.ipv4.ip_nonlocal_bind=1" >> /etc/sysctl.conf
sysctl -p

三、使用kubeasz安装K8s集群

kubeasz项目官方文档

部署步骤

以下示例创建一个4节点的多主高可用集群，文档中命令默认都需要root权限运行。

1.基础系统配置

  • 2c/4g内存/40g硬盘（该配置仅测试用）
  • 最小化安装Ubuntu 16.04 server或者CentOS 7 Minimal
  • 配置基础网络、更新源、SSH登录等

2.在每个节点安装依赖工具

推荐使用ansible in docker 容器化方式运行，无需安装额外依赖。

3.准备ssh免密登陆

配置从部署节点能够ssh免密登陆所有节点，并且设置python软连接

#$IP为所有节点地址包括自身，按照提示输入yes 和root密码
ssh-copy-id $IP 

# 为每个节点设置python软链接
ssh $IP ln -s /usr/bin/python3 /usr/bin/python

4.在部署节点编排k8s安装

  • 4.1 下载项目源码、二进制及离线镜像

下载工具脚本ezdown，举例使用kubeasz版本3.3.1

export release=3.3.1
wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
chmod +x ./ezdown

下载kubeasz代码、二进制、默认容器镜像（更多关于ezdown的参数，运行./ezdown 查看）

# 国内环境
./ezdown -D
# 海外环境
#./ezdown -D -m standard

【可选】下载额外容器镜像（cilium,flannel,prometheus等）

./ezdown -X

【可选】下载离线系统包 (适用于无法使用yum/apt仓库情形)

./ezdown -P

上述脚本运行成功后，所有文件（kubeasz代码、二进制、离线镜像）均已整理好放入目录/etc/kubeasz

  • 4.2 创建集群配置实例
# 容器化运行kubeasz
./ezdown -S

# 创建新集群 k8s-01
docker exec -it kubeasz ezctl new k8s-01
2021-01-19 10:48:23 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-01
2021-01-19 10:48:23 DEBUG set version of common plugins
2021-01-19 10:48:23 DEBUG cluster k8s-01: files successfully created.
2021-01-19 10:48:23 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-01/hosts'
2021-01-19 10:48:23 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-01/config.yml'

然后根据提示配置'/etc/kubeasz/clusters/k8s-01/hosts' 和 '/etc/kubeasz/clusters/k8s-01/config.yml'：根据前面节点规划修改hosts 文件和其他集群层面的主要配置选项；其他集群组件等配置项可以在config.yml 文件中修改。

  • 4.3 开始安装
    如果你对集群安装流程不熟悉，请阅读项目首页 安装步骤 讲解后分步安装，并对 每步都进行验证
#建议配置命令alias，方便执行
echo "alias dk='docker exec -it kubeasz'" >> /root/.bashrc
source /root/.bashrc

# 一键安装，等价于执行docker exec -it kubeasz ezctl setup k8s-01 all
dk ezctl setup k8s-01 all

# 或者分步安装，具体使用 dk ezctl help setup 查看分步安装帮助信息
# dk ezctl setup k8s-01 01
# dk ezctl setup k8s-01 02
# dk ezctl setup k8s-01 03
# dk ezctl setup k8s-01 04
...

个性化集群参数配置

kubeasz创建集群主要在以下两个地方进行配置：(假设集群名xxxx)

  • clusters/xxxx/hosts 文件（模板在example/hosts.multi-node）：集群主要节点定义和主要参数配置、全局变量
  • clusters/xxxx/config.yml（模板在examples/config.yml）：其他参数配置或者部分组件附加参数

clusters/xxxx/hosts (ansible hosts)

主要包括集群节点定义和集群范围的主要参数配置

  • 尽量保持配置简单灵活
  • 尽量保持配置项稳定

常用设置项：

  • 修改容器运行时: CONTAINER_RUNTIME="containerd"
  • 修改集群网络插件：CLUSTER_NETWORK="calico"
  • 修改容器网络地址：CLUSTER_CIDR="192.168.0.0/16"
  • 修改NodePort范围：NODE_PORT_RANGE="30000-32767"

clusters/xxxx/config.yml

主要包括集群某个具体组件的个性化配置，具体组件的配置项可能会不断增加；可以在不做任何配置更改情况下使用默认值创建集群

根据实际需要配置 k8s 集群，常用举例

  • 配置使用离线安装系统包：INSTALL_SOURCE: "offline" （需要ezdown -P 下载离线系统软件）
  • 配置CA证书以及其签发证书的有效期
  • 配置 apiserver 支持公网域名：MASTER_CERT_HOSTS
  • 配置 cluster-addon 组件安装
  • ...
Powered by Gridea RSS