秋栈博客

七月

首页Hello HaloHalo关于

POD直通

14
2023-12-19

https://github.com/FRRouting/frr/releases

http://docs.frrouting.org/en/stable-8.5/bgp.html

网段规划

集群节点IP： 10.10.0.0/16；
Pod Network CIDR： 10.254.0.0/16
Service IPs： 10.253.0.0/16

内核参数调整

#net.ipv4.conf.default.rp_filter=1
#net.ipv4.conf.all.rp_filter=1

net.ipv4.ip_forward=1
net.ipv4.conf.all.accept_redirects = 1
net.ipv6.conf.all.accept_redirects = 1
# _or_
# Accept ICMP redirects only for gateways listed in our default
# gateway list (enabled by default)
net.ipv4.conf.all.secure_redirects = 1

# Do not send ICMP redirects (we are a router)
net.ipv4.conf.all.send_redirects = 1

# Do not accept IP source route packets (we are a router)
net.ipv4.conf.all.accept_source_route = 1
net.ipv6.conf.all.accept_source_route = 1

# Log Martian Packets
net.ipv4.conf.all.log_martians = 1

物理机调整

全局关闭网卡防欺诈

安装frr服务

curl -s https://deb.frrouting.org/frr/keys.gpg | tee /usr/share/keyrings/frrouting.gpg > /dev/null
FRRVER="frr-stable"
echo deb '[signed-by=/usr/share/keyrings/frrouting.gpg]' https://deb.frrouting.org/frr $(lsb_release -s -c) $FRRVE\
R | tee -a /etc/apt/sources.list.d/frr.list
apt update && apt install frr frr-pythontools

打开BGP功能

vim /etc/frr/daemons
bgpd=yes

systemctl restart frr.service

通告网段

10-10-255-254# configure
router bgp 65001
bgp router-id 192.168.1.251
timers bgp 3 9
neighbor cluster-group peer-group
neighbor cluster-group remote-as 65001
neighbor cluster-group advertisement-interval 0
neighbor cluster-group timers connect 5
bgp listen limit 1000
bgp listen range 10.10.0.0/16 peer-group cluster-group
bgp listen range 192.168.1.0/24 peer-group cluster-group
address-family ipv4 unicast
neighbor cluster-group route-reflector-client
neighbor cluster-group next-hop-self force
exit-address-family
exit
exit
write

集群

wget https://lcc-init.oss-cn-hangzhou.aliyuncs.com/sealos/sealos_4.2.3_linux_amd64.tar.gz && \
tar zxvf sealos_4.2.3_linux_amd64.tar.gz sealos && \
chmod +x sealos && mv sealos /usr/bin && \
rm -rf sealos_4.2.3_linux_amd64.tar.gz && \
sealos completion bash > /etc/bash_completion.d/sealos.bash && \
source /etc/bash_completion.d/sealos.bash

cat <<EOF >> /root/.bashrc 
source <(helm completion bash)
source <(kubectl completion bash)
alias km='kubecm'
alias kms='kubecm switch'
alias kma='kubecm add'
alias kmn='kubecm namespace'
alias k='kubectl'
alias ke='k edit'
alias kg='k get'
alias kga='k get all'
alias kgat="kga -o json | jq '.items[] | {name: .metadata.name, node: .spec.nodeName, status: .status.phase}'"
alias kgd='k get deploy'
alias kgn='k get nodes'
alias kgnt="kgn -o json | jq '.items[] | {name: .metadata.name, taints: .spec.taints}'"
alias wkg='watch -d -n1 kubectl get'
alias wkgp='watch -d -n1 kubectl get pods'
alias kd='k describe'
alias kl='k logs'
alias kc='k create'
alias kgp='k get pods'
alias kgpt="kgp -o json | jq '.items[] | {name: .metadata.name, node: .spec.nodeName, status: .status.phase}'"
alias kx='k exec -it'
alias kxsh='k exec -it -- sh'
alias kxbh='k exec -it -- bash'
EOF
source /root/.bashrc

设置Calico使用BGP模式

$ kubectl edit ds calico-node -n calico-system
        ......
        - name: CALICO_IPV4POOL_IPIP
          value: Never
        ## 新增 CALICO_AUTODETECTION_METHOD，interface为节点业务出接口
        - name: CALICO_AUTODETECTION_METHOD
          value: interface=eth0
bgp_peer.yaml
cat <<EOF > bgp_peer.yaml
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: to-rr-254
spec:
  peerIP: 10.10.255.254
  asNumber: 65001
EOF

kubectl apply -f bgp_peer.yaml
bgp_config.yaml
cat <<EOF > bgp_config.yaml
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
  name: default
spec:
  logSeverityScreen: Info
  nodeToNodeMeshEnabled: false
  asNumber: 65001
  serviceClusterIPs:
    - cidr: 10.253.24.0/21
  bindMode: NodeIP
  communities:
    - name: default-cluster-community
      value: 65001:100
  prefixAdvertisements:
    - cidr: 10.253.24.0/21
      communities:
         - default-cluster-community
    - cidr: 10.254.24.0/21
      communities:
         - default-cluster-community
EOF

kubectl apply -f bgp_config.yaml
双RR高可用bgp_peer.yaml
---
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: to-rr-253
spec:
  peerIP: 10.10.255.253
  asNumber: 65001
---
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
  name: to-rr-254
spec:
  peerIP: 10.10.255.254
  asNumber: 65001

登录路由器检查

10-10-255-254# show ip bgp summary

登录Master检查

wget https://lcc-init.oss-cn-hangzhou.aliyuncs.com/calicoctl && \
mv calicoctl /usr/bin/calicoctl && chmod +x /usr/bin/calicoctl && \
calicoctl node status

配置负载均衡器

# 如果kube-proxy使用的是ipvs模式，需要修改kube-proxy配置文件，启用严格的ARP
kubectl edit configmap -n kube-system kube-proxy

apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: "ipvs"
ipvs:
  strictARP: true
......

安装Metallb

# 物理网络不支持BGP，所以这里使用原生的L2模式
wget https://lcc-init.oss-cn-hangzhou.aliyuncs.com/metallb/v0.13.10/metallb-native.yaml
wget https://lcc-init.oss-cn-hangzhou.aliyuncs.com/metallb/v0.13.10/metallb-frr.yaml
kubectl apply -f metallb-native.yaml && sleep 5 && kubectl apply -f metallb-frr.yaml

创建LB地址池

cat <<EOF > metallb-pool-d.yaml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: metallb-pool-d
  namespace: metallb-system
spec:
  addresses:
  - 10.10.255.151-10.10.255.200
EOF

kubectl apply -f metallb-pool-d.yaml

创建 L2Advertisement，并关联 IPAdressPool

# LB的 EXTERNAL-IP也是模拟出来的IP地址，在Node上不存在，所以需要L2Advertisement响应本地网络arp请求，以将计算机的MAC地址提供给客户端。
cat <<EOF > metallb-l2-ad.yaml
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: metallb-l2-ad
  namespace: metallb-system
spec:
  ipAddressPools:
  - metallb-pool-d
EOF

kubectl apply -f metallb-l2-ad.yaml

验证

# Metallb 会按照 Service为单位选择Leader Node，而不是整体选举一个Leader，也就是说存在多个Service的情况下，LB工作的Node会分散在集群不同的Node之上

网段划分

PodSubnet

10.254.0.0/16 划分为32个子网，每个子网有2046个可用IP 掩码：255.255.248.0

Cluster-A：10.254.0.0/21 # 可用 IP 范围: 10.254.0.1 - 10.254.7.254
Cluster-B：10.254.8.0/21 # 可用 IP 范围: 10.254.8.1 - 10.254.15.254
Cluster-C：10.254.16.0/21 # 可用 IP 范围: 10.254.16.1 - 10.254.23.254
Cluster-D：10.254.24.0/21 # 可用 IP 范围: 10.254.24.1 - 10.254.31.254
Cluster-E：10.254.32.0/21 # 可用 IP 范围: 10.254.32.1 - 10.254.39.254

ServiceSubnet

10.253.0.0/16 划分为32个子网，每个子网有2046个可用IP 掩码：255.255.248.0

Cluster-A：10.253.0.0/21 # 可用 IP 范围: 10.253.0.1 - 10.253.7.254
Cluster-B：10.253.8.0/21 # 可用 IP 范围: 10.253.8.1 - 10.253.15.254
Cluster-C：10.253.16.0/21 # 可用 IP 范围: 10.253.16.1 - 10.253.23.254
Cluster-D：10.253.24.0/21 # 可用 IP 范围: 10.253.24.1 - 10.253.31.254
Cluster-E：10.253.32.0/21 # 可用 IP 范围: 10.253.32.1 - 10.253.39.254

LB Pool

Cluster-A：10.10.255.1-10.10.255.50
Cluster-B：10.10.255.51-10.10.255.100
Cluster-C：10.10.255.101-10.10.255.150
Cluster-D：10.10.255.151-10.10.255.200
Cluster-E：10.10.255.201-10.10.255.255

useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash
gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner
gitlab-runner start
  • 0
Hello Halo