异地云服务器部署k8s集群

初始化系统

升级内核:https://www.chenmx.net/?p=208

# 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

# 关闭selinux
sed -i 's/enforcing/disabled/' /etc/selinux/config # 永久
setenforce 0 # 临时

# 关闭swap
swapoff -a # 临时
sed -ri 's/.*swap.*/#&/' /etc/fstab # 永久

# 根据规划设置主机名 hostname:名称,方便记忆
hostnamectl set-hostname hostname

# 在master添加hosts
cat >> /etc/hosts << EOF
82.156.215.56 k8s-master #k8smaster->上文中的hostname
124.71.156.166 k8s-node #同理
49.232.0.68 k8s-node1 #同理
EOF

modprobe br_netfilter

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack #内核版本小于4.19 nf_conntrack_ipv4
EOF

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack

#云服务器一般不需要
# 设置系统时区为 中国/上海
timedatectl set-timezone Asia/Shanghai
# 将当前的UTC时间写入硬件时钟
timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务
systemctl restart rsyslog
systemctl restart crond

添加阿里云YUM软件源

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

安装​kubeadm​​​,​​kubelet​​​和​​kublinux是什么操作系统ectl​linux是什么操作系统

yum install -y kubelet-1.21.0 kubeadm-1.21.0 kubectl-1.21.0
systemctl enable kubelet
#查看版本信息
kubectl version
kubeadm version
kubelet --version

安装docker,准备相应镜像

  • 各节点需ide0sgrtsts另类要的镜像
    #!/bin/bash
    #pull images

    ver=v1.21.0
    registry=registry.cn-hangzhou.aliyuncs.com/google_containers
    images=`kubeadm config images list --kubernetes-version=$ver |awk -F '/' '{print $2}'`

    for image in $images
    do
    if [ $image != coredns ];then
    docker pull ${registry}/$image
    if [ $? -eq 0 ];then
    docker tag ${registry}/$image k8s.gcr.io/$image
    docker rmi ${registry}/$image
    else
    echo "ERROR: 下载镜像报错,$image"
    fi
    else
    docker pull coredns/coredns:1.8.0
    docker tag coredns/coredns:1.8.0 k8s.gcr.io/coredns/coredns:v1.8.0
    docker rmi coredns/coredns:1.8.0
    fi
    done

    chmod +x pullimages.sh && ./pullimages.sh
    • 安装并配置docker
      https://www.chenmx.net/?p=31

      建立虚拟网卡(所以节点)

      cat > /etc/sysconfig/network-scripts/ifcfg-eth0:1 <<EOF
      BOOTPROTO=static
      DEVICE=eth0:1
      IPADDR=82.156.215.56 #你的公网IP
      PREFIX=32
      TYPE=Ethernet
      USERCTL=no
      ONBOOT=yes
      EOF
      #重启网卡
      systemctl restart network
      查看全部IP
      ip addr

      修改​​kubelet​​启动参数

      vim /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf

      [Service]
      Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
      Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
      # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
      EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
      # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
      # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
      EnvironmentFile=-/etc/sysconfig/kubelet
      ExecStart=
      ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS --node-ip=82.156.215.56

      使用​​kub恶毒男配只想c位出道eadm​初始化主节点(master节点恶毒男配不争了

      执行上文中下载docker镜像的脚本

      # step1 添加配置文件,注意替换下面的IP
      cat > kubeadm-config.yaml <<EOF
      apiVersion: kubeadm.k8s.io/v1beta2
      kind: ClusterConfiguration
      kubernetesVersion: v1.21.0
      apiServer:
      certSANs:
      - k8s-master #请替换为hostname
      - 82.156.215.56 #请替换为公网
      - 82.156.215.56
      - 10.96.0.1 #不要替换,此IP是API的集群地址,部分服务会用到
      controlPlaneEndpoint: 82.156.215.56:6443 #替换为公网IP
      networking:
      podSubnet: 10.244.0.0/16
      serviceSubnet: 10.96.0.0/12
      --- 将默认调度方式改为ipvs
      apiVersion: kubeproxy-config.k8s.io/v1alpha1
      kind: KubeProxyConfiguration
      featureGates:
      SupportIPVSProxyMode: true
      mode: ipvs
      EOF

      #初始化 如果是1核心或者1G内存的请在末尾添加参数(--ignore-preflight-errors=all),否则会初始化失败
      kubeadm init --config=kubeadm-config.yaml --ignore-preflight-errors=all

      #上面初始化成功后,将会生成kubeconfig文件,用于请求api服务器,请执行下面操作
      mkdir -p $HOME/.kube
      sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
      sudo chown $(id -u):$(id -g) $HOME/.kube/config

      #node点加入主节点使用
      kubeadm join 82.156.215.56:6443 --token kias9b.0sngusi94r8bh6f6 --discovery-token-ca-cert-hash sha256:09868a449efed1bb017d3a7b6e7fc3386feac6fcbc076350e5868ff1fc5be3f5

      ,遇到这种问题只需要​​sysctl -w net.ipv4.ip_forward=1​​即可解决

      修改​初始化失败是怎么解决​kube-apiserver​​参数(主节点)

      vim /etc/kubernetes/manifests/kube-apiserver.yaml

      apiVersion: v1
      kind: Pod
      metadata:
      annotations:
      kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint: 82.156.215.56:6443
      creationTimestamp: null
      labels:
      component: kube-apiserver
      tier: control-plane
      name: kube-apiserver
      namespace: kube-system
      spec:
      containers:
      - command:
      - kube-apiserver
      - --advertise-address=82.156.218.219
      - --bind-address=0.0.0.0
      - --allow-privileged=true
      - --authorization-mode=Node,RBAC
      - --client-ca-file=/etc/kubernetes/pki/ca.crt
      - --enable-admission-plugins=NodeRestriction
      - --enable-bootstrap-token-auth=true
      - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
      - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
      - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
      - --etcd-servers=https://127.0.0.1:2379
      - --insecure-port=0
      - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
      - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
      - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
      - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
      - --requestheader-allowed-names=front-proxy-client
      - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
      - --requestheader-extra-headers-prefix=X-Remote-Extra-
      - --requestheader-group-headers=X-Remote-Group
      - --requestheader-username-headers=X-Remote-User
      - --secure-port=6443
      - --service-account-issuer=https://kubernetes.default.svc.cluster.local
      - --service-account-key-file=/etc/kubernetes/pki/sa.pub
      - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
      - --service-cluster-ip-range=10.96.0.0/12
      - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
      - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
      image: k8s.gcr.io/kube-apiserver:v1.21.0
      imagePullPolicy: IfNotPresent
      livenessProbe:
      failureThreshold: 8
      httpGet:
      host: 82.156.215.56
      path: /livez
      port: 6443
      scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
      name: kube-apiserver
      readinessProbe:
      failureThreshold: 3
      httpGet:
      host: 82.156.215.56
      path: /readyz
      port: 6443
      scheme: HTTPS
      periodSeconds: 1
      timeoutSeconds: 15
      resources:
      requests:
      cpu: 250m
      startupProbe:
      failureThreshold: 24
      httpGet:
      host: 82.156.215.56
      path: /livez
      port: 6443
      scheme: HTTPS
      initialDelaySeconds: 10
      periodSeconds: 10
      timeoutSeconds: 15
      volumeMounts:
      - mountPath: /etc/ssl/certs
      name: ca-certs
      readOnly: true
      - mountPath: /etc/pki
      name: etc-pki
      readOnly: true
      - mountPath: /etc/kubernetes/pki
      name: k8s-certs
      readOnly: true
      hostNetwork: true
      priorityClassName: system-node-critical
      volumes:
      - hostPath:
      path: /etc/ssl/certs
      type: DirectoryOrCreate
      name: ca-certs
      - hostPath:
      path: /etc/pki
      type: DirectoryOrCreate
      name: etc-pki
      - hostPath:
      path: /etc/kubernetes/pki
      type: DirectoryOrCreate
      name: k8s-certs
      status: {}

      加入恶毒男配在线养崽node节点

      kubeadm join 82.156.215.56:6443 --token kias9b.0sngusi94r8bh6f6 --discovery-token-ca-cert-hash sha256:09868a449efed1bb017d3a7b6e7fc3386feac6fcbc076350e5868ff1fc5be3f5

      #master节点执行,看是否添加成功 ip地址等信息
      kubectl get nodes -o wide

      #默认token有效期为24小时,当过期之后,该token就不可用了。这时就需要重新创建token
      kubeadm token create --print-join-command

      修改​​flannel​​文件并安装(master节点)

      #下载文件
      wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

      # 共修改两个地方,一个是args下,添加
      args:
      - --public-ip=$(PUBLIC_IP) # 添加此参数,申明公网IP
      - --iface=eth0 # 添加此参数,绑定网卡


      # 然后是env下
      env:
      - name: PUBLIC_IP #添加环境变量
      valueFrom:
      fieldRef:
      fieldPath: status.podIP


      ---
      apiVersion: policy/v1beta1
      kind: PodSecurityPolicy
      metadata:
      name: psp.flannel.unprivileged
      annotations:
      seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
      seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
      apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
      apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
      spec:
      privileged: false
      volumes:
      - configMap
      - secret
      - emptyDir
      - hostPath
      allowedHostPaths:
      - pathPrefix: "/etc/cni/net.d"
      - pathPrefix: "/etc/kube-flannel"
      - pathPrefix: "/run/flannel"
      readOnlyRootFilesystem: false
      # Users and groups
      runAsUser:
      rule: RunAsAny
      supplementalGroups:
      rule: RunAsAny
      fsGroup:
      rule: RunAsAny
      # Privilege Escalation
      allowPrivilegeEscalation: false
      defaultAllowPrivilegeEscalation: false
      # Capabilities
      allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
      defaultAddCapabilities: []
      requiredDropCapabilities: []
      # Host namespaces
      hostPID: false
      hostIPC: false
      hostNetwork: true
      hostPorts:
      - min: 0
      max: 65535
      # SELinux
      seLinux:
      # SELinux is unused in CaaSP
      rule: 'RunAsAny'
      ---
      kind: ClusterRole
      apiVersion: rbac.authorization.k8s.io/v1
      metadata:
      name: flannel
      rules:
      - apiGroups: ['extensions']
      resources: ['podsecuritypolicies']
      verbs: ['use']
      resourceNames: ['psp.flannel.unprivileged']
      - apiGroups:
      - ""
      resources:
      - pods
      verbs:
      - get
      - apiGroups:
      - ""
      resources:
      - nodes
      verbs:
      - list
      - watch
      - apiGroups:
      - ""
      resources:
      - nodes/status
      verbs:
      - patch
      ---
      kind: ClusterRoleBinding
      apiVersion: rbac.authorization.k8s.io/v1
      metadata:
      name: flannel
      roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: flannel
      subjects:
      - kind: ServiceAccount
      name: flannel
      namespace: kube-system
      ---
      apiVersion: v1
      kind: ServiceAccount
      metadata:
      name: flannel
      namespace: kube-system
      ---
      kind: ConfigMap
      apiVersion: v1
      metadata:
      name: kube-flannel-cfg
      namespace: kube-system
      labels:
      tier: node
      app: flannel
      data:
      cni-conf.json: |
      {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
      {
      "type": "flannel",
      "delegate": {
      "hairpinMode": true,
      "isDefaultGateway": true
      }
      },
      {
      "type": "portmap",
      "capabilities": {
      "portMappings": true
      }
      }
      ]
      }
      net-conf.json: |
      {
      "Network": "10.244.0.0/16",
      "Backend": {
      "Type": "vxlan"
      }
      }
      ---
      apiVersion: apps/v1
      kind: DaemonSet
      metadata:
      name: kube-flannel-ds
      namespace: kube-system
      labels:
      tier: node
      app: flannel
      spec:
      selector:
      matchLabels:
      app: flannel
      template:
      metadata:
      labels:
      tier: node
      app: flannel
      spec:
      affinity:
      nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
      nodeSelectorTerms:
      - matchExpressions:
      - key: kubernetes.io/os
      operator: In
      values:
      - linux
      hostNetwork: true
      priorityClassName: system-node-critical
      tolerations:
      - operator: Exists
      effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
      image: quay.io/coreos/flannel:v0.14.0
      command:
      - cp
      args:
      - -f
      - /etc/kube-flannel/cni-conf.json
      - /etc/cni/net.d/10-flannel.conflist
      volumeMounts:
      - name: cni
      mountPath: /etc/cni/net.d
      - name: flannel-cfg
      mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
      image: quay.io/coreos/flannel:v0.14.0
      command:
      - /opt/bin/flanneld
      args:
      - --ip-masq
      - --public-ip=$(PUBLIC_IP)
      - --iface=eth0
      - --kube-subnet-mgr
      args:
      - --ip-masq
      - --kube-subnet-mgr
      resources:
      requests:
      cpu: "100m"
      memory: "50Mi"
      limits:
      cpu: "100m"
      memory: "50Mi"
      securityContext:
      privileged: false
      capabilities:
      add: ["NET_ADMIN", "NET_RAW"]
      env:
      - name: PUBLIC_IP
      valueFrom:
      fieldRef:
      fieldPath: status.podIP
      - name: POD_NAME
      valueFrom:
      fieldRef:
      fieldPath: metadata.name
      - name: POD_NAMESPACE
      valueFrom:
      fieldRef:
      fieldPath: metadata.namespace
      volumeMounts:
      - name: run
      mountPath: /run/flannel
      - name: flannel-cfg
      mountPath: /etc/kube-flannel/
      volumes:
      - name: run
      hostPath:
      path: /run/flannel
      - name: cni
      hostPath:
      path: /etc/cni/net.d
      - name: flannel-cfg
      configMap:
      name: kube-flannel-cfg

      #创建flannel
      kubectl apply -f kube-flannel.yaml

      #检查pod是否都已经在running状态
      kubectl get pods -o wide --all-namespaces

      手动开启配置,开启​​ipvs​​转发模式(master节点)

      kubectl edit configmaps -n kube-system kube-proxy

      # Please edit the object below. Lines beginning with a '#' will be ignored,
      # and an empty file will abort the edit. If an error occurs while saving this file will be
      # reopened with the relevant failures.
      #
      apiVersion: v1
      data:
      config.conf: |-
      apiVersion: kubeproxy.config.k8s.io/v1alpha1
      bindAddress: 0.0.0.0
      bindAddressHardFail: false
      clientConnection:
      acceptContentTypes: ""
      burst: 0
      contentType: ""
      kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
      qps: 0
      clusterCIDR: 10.244.0.0/16
      configSyncPeriod: 0s
      conntrack:
      maxPerCore: null
      min: null
      tcpCloseWaitTimeout: null
      tcpEstablishedTimeout: null
      detectLocalMode: ""
      enableProfiling: false
      healthzBindAddress: ""
      hostnameOverride: ""
      iptables:
      masqueradeAll: false
      masqueradeBit: null
      minSyncPeriod: 0s
      syncPeriod: 0s
      ipvs:
      excludeCIDRs: null
      minSyncPeriod: 0s
      scheduler: ""
      strictARP: false
      syncPeriod: 0s
      tcpFinTimeout: 0s
      tcpTimeout: 0s
      udpTimeout: 0s
      kind: KubeProxyConfiguration
      metricsBindAddress: ""
      mode: "ipvs"
      nodePortAddresses: null
      oomScoreAdj: null
      portRange: ""
      showHiddenMetricsForVersion: ""
      udpIdleTimeout: 0s
      winkernel:
      enableDSR: false
      networkName: ""
      sourceVip: ""
      kubeconfig.conf: |-
      apiVersion: v1
      ....

      搞了2个星期终于搞好了,如果舍得花钱弄slb或者本地内网环境ideological通的情况是非常简单的,怎奈一个穷啊…

      欢迎大家访问我初始化数组​​个人小站​​:https://www.chenmx.net,获取更多有趣的博文!