为k8s集群手动添加节点脚本

k8s手动添加节点脚本

#!/usr/bin/env bash

################################################
#
# KUBE_VERSION     the expected kubernetes version
# eg.  ./attachKubernetesNode
#           --docker-version 17.06.2-ce-1 
#           --token 264db1.30bcc2b89969a4ca 
#           --endpoint 192.168.0.80:6443
#           --cluster-dns 172.19.0.10
################################################

set -e -x

PKG=pkg

export CLOUD_TYPE=private
export openapi="http://cs-intranet.inter.env17e.shuguang.com"
export GPU_FOUNDED=0
export PKG_FILE_SERVER="https://chest-biz.acs.inter.env17e.shuguang.com:8080/static/k8s/"
export KUBE_REPO_PREFIX="registry.acs.intra.env17e.shuguang.com/acs"
export INTERNET_DOMAIN="inter.env17e.shuguang.com"
export SLB_ENDPOINT="http://slb-vpc.inter.env17e.shuguang.com"
export ECS_ENDPOINT="http://ecs-internal.inter.env17e.shuguang.com"
export INSECURE_REGISTRY="cr.registry.inter.env17e.shuguang.com"

public::common::log() {
        echo $(date +"[%Y%m%d %H:%M:%S]: ") $1
}

public::cert::addCA()
{
    set +e
    curl -k --retry 4 --retry-delay 5 -sSL $PKG_FILE_SERVER/$CLOUD_TYPE/static.pem>static.pem
    if [ "$?" != 0 ];then
        echo "failed to download root ca from static file server."; set -e
        retrun
    fi
    set -e
    dir=/etc/pki/ca-trust/source/anchors/
    mkdir -p $dir ; mv static.pem $dir/static.pem
    update-ca-trust
}

public::common::arch() {
    export OS_ARCH=amd64
    case `uname -m` in
        "sw_64" )
            export OS_ARCH=sw64
            ;;
        "x86_64")
            export OS_ARCH=amd64
            ;;
        "arm64")
            export OS_ARCH=arm64
            ;;
        *)
            echo "unknown arch `uname -a`, default to amd64 "
            ;;
    esac
        OS_TYPE=linux
}
public::common::arch

public::common::region()
{
    if [ "$CLOUD_TYPE" != "agility" ];then
        region=$(curl --retry 5  -sSL http://100.100.100.200/latest/meta-data/region-id)
        if [ "" == "$region" ];then
            kube::common::log "can not get regionid and instanceid! 
                curl --retry 5 -sSL http://100.100.100.200/latest/meta-data/region-id" && exit 256
                fi
                export REGION=$region
        else
                public::common::log "Do nothing for aglity"
        fi
}

public::common::os_env() {
        ubu=$(cat /etc/issue | grep "Ubuntu 16.04" | wc -l)
        cet=$(cat /etc/centos-release | grep "CentOS" | wc -l)
        suse=$(cat /etc/os-release | grep "SUSE" | wc -l)
        redhat=$(cat /etc/redhat-release | grep "Red Hat" | wc -l)
        alios=$(cat /etc/redhat-release | grep "Alibaba" | wc -l)
        if [ "$ubu" == "1" ]; then
                export OS="Ubuntu"
        elif [ "$cet" == "1" ]; then
                export OS="CentOS"
        elif [ "$suse" == "1" ]; then
                export OS="SUSE"
        elif [ "$redhat" == "1" ]; then
                export OS="RedHat"
        elif [ "$alios" == "1" ]; then
                export OS="AliOS"
        else
                public::common::log "unkown os...   exit"
                exit 1
        fi
}

## public::common::prepare_package
#  New Package Name Pattern to support multi-os & multi-arch
#       OS_TYPE:   linux[CentOS(RPM)]
#       Arch: amd64 sw64 arm64
#  $PKG_FILE_SERVER/$CLOUD_TYPE/pkg/$PKG_TYPE/${PKG_TYPE}-${PKG_VERSION}-${OS_TYPE}-${OS_ARCH}.tar.gz
public::common::prepare_package() {
        PKG_TYPE=$1
        PKG_VERSION=$2
        if [[ "$PKG_VERSION" != "1.11.5" ]];
    then
        MULTI_OS=-${OS_TYPE}-${OS_ARCH}
    fi
        pkg=${PKG_TYPE}-${PKG_VERSION}${MULTI_OS}.tar.gz
        if [[ ! -f ${pkg} ]]; then
                if [[ -z ${PKG_FILE_SERVER} ]]; then
                        public::common::log "local file ${pkg} does not exist, And PKG_FILE_SERVER is not config"
                        public::common::log "installer does not known where to download installer binary package without PKG_FILE_SERVER env been set. Error: exit"
                        exit 1
                fi
                public::common::log "local file ${pkg} does not exist, trying to download from [$PKG_FILE_SERVER]"
                curl -k --retry 4 --retry-delay 5 -sSL 
                    ${PKG_FILE_SERVER}/${CLOUD_TYPE}/pkg/${PKG_TYPE}/${pkg} 
                        >${pkg} || (public::common::log "download failed with 4 retry,exit 1" && exit 1)
        fi
        tar -xvf ${pkg} || (public::common::log "untar ${pkg} failed!, exit" && exit 1)
}

public::common::nodeid()
{
    if [ "$CLOUD_TYPE" != "agility" ];then
        region=$(curl --retry 5  -sSL http://100.100.100.200/latest/meta-data/region-id)
        insid=$(curl --retry 5  -sSL http://100.100.100.200/latest/meta-data/instance-id)
        if [ "" == "$region" -o "" == "$insid" ];then
            kube::common::log "can not get regionid and instanceid! 
            curl --retry 5 -sSL http://100.100.100.200/latest/meta-data/region-id" && exit 256
                fi
                export NODE_ID=$region.$insid
        else
                public::common::log "Do nothing for aglity"
        fi
}

function version_gt() { test "$(echo "$@" | tr " " "
" | sort -V | head -n 1)" != "$1"; }

function retry() {
        local n=0
        local try=$1
        local cmd="${@:2}"
        [[ $# -le 1 ]] && {
                echo "Usage $0 <retry_number> <Command>"
        }
        set +e
        until
                [[ $n -ge $try ]]
        do
                $cmd && break || {
                        echo "Command Fail.."
                        ((n++))
                        echo "retry $n :: [$cmd]"
                        sleep 2
                }
        done
        set -e
}

public::common::install_package() {

        public::docker::install

        public::common::prepare_package "kubernetes" $KUBE_VERSION
        public::common::prepare_package "addons" $KUBE_VERSION || true

    local registry_domain=`echo $KUBE_REPO_PREFIX|awk -F "/" '{print $1}'`
    sed -i "s#registry.cn-hangzhou.aliyuncs.com#$registry_domain#g" pkg/kubernetes/$KUBE_VERSION/module/*

    public::common::apply_sysctls
    # Install crictl
        CRI_CRL="pkg/kubernetes/$KUBE_VERSION/rpm/crictl"
        if [ -f "$CRI_CRL" ];then
        chmod +x "$CRI_CRL"
        mv "$CRI_CRL" /usr/bin/crictl
        fi
    if [ "$OS" == "CentOS" ] || [ "$OS" == "RedHat" ] || [ "$OS" == "AliOS" ];then
        dir=pkg/kubernetes/$KUBE_VERSION/rpm

        # Install nfs client and socat
        rpm --replacepkgs -ivh `ls $dir | xargs -I '{}' echo -n "$dir/{} "`
    elif [ "$OS" == "Ubuntu" ];then
        dir=pkg/kubernetes/$KUBE_VERSION/debain
        dpkg -i `ls $dir | xargs -I '{}' echo -n "$dir/{} "`
    elif [ "$OS" == "SUSE" ];then
        dir=pkg/kubernetes/$KUBE_VERSION/rpm
        zypper --no-gpg-checks install -y `ls $dir | xargs -I '{}' echo -n "$dir/{} "`
    fi

        sed -i "s#--cluster-dns=10.96.0.10 --cluster-domain=cluster.local#--cluster-dns=$CLUSTER_DNS 
    --pod-infra-container-image=$KUBE_REPO_PREFIX/pause-amd64:3.0 
    --enable-controller-attach-detach=false 
    --cluster-domain=cluster.local --cloud-provider=external 
    --hostname-override=$NODE_ID --provider-id=$NODE_ID#g" 
                /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
        if docker info | grep 'Cgroup Driver' | grep cgroupfs; then
                sed -i -e 's/cgroup-driver=systemd/cgroup-driver=cgroupfs/g' 
                        /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
        fi

        # set node resource reserve when available memory > 3.5G
        if [[ $(cat /proc/meminfo | awk '/MemTotal/ {print $2}') -gt $((1024 * 512 * 7)) ]]; then
                sed -i -e "s/--cgroup-driver=/--system-reserved=memory=300Mi 
                --kube-reserved=memory=400Mi 
                --eviction-hard=imagefs.available<15%,memory.available<300Mi,nodefs.available<10%,nodefs.inodesFree<5% --cgroup-driver=/g" 
                        /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
        fi

        systemctl daemon-reload
        systemctl enable kubelet.service
        systemctl start kubelet.service
}

public::common::apply_sysctls() {
        declare -A sysctls_map=(
                ["vm.max_map_count"]="262144"
                ["kernel.softlockup_panic"]="1"
                ["kernel.softlockup_all_cpu_backtrace"]="1"
                ["net.core.somaxconn"]="32768"
                ["net.core.rmem_max"]="16777216"
                ["net.core.wmem_max"]="16777216"
                ["net.ipv4.tcp_wmem"]="4096 12582912 16777216"
                ["net.ipv4.tcp_rmem"]="4096 12582912 16777216"
                ["net.ipv4.tcp_max_syn_backlog"]="8096"
                ["net.ipv4.tcp_slow_start_after_idle"]="0"
                ["net.core.netdev_max_backlog"]="16384"
                ["fs.file-max"]="2097152"
                ["fs.inotify.max_user_instances"]="8192"
                ["fs.inotify.max_user_watches"]="524288"
                ["fs.inotify.max_queued_events"]="16384"
                ["net.ipv4.ip_forward"]="1"
                ["net.bridge.bridge-nf-call-iptables"]="1"
                ["fs.may_detach_mounts"]="1"
        )

        if [ ! -f /etc/sysctl.d/99-k8s.conf ]; then
                mkdir -p /etc/sysctl.d/ && touch /etc/sysctl.d/99-k8s.conf
                echo "#sysctls for k8s node config" >/etc/sysctl.d/99-k8s.conf
        fi
        for key in ${!sysctls_map[@]}; do
                sysctl_path="/proc/sys/"${key//.//}
                if [ -f ${sysctl_path} ]; then
                        sed -i "/${key}/ d" /etc/sysctl.d/99-k8s.conf
                        echo "${key}=${sysctls_map[${key}]}" >>/etc/sysctl.d/99-k8s.conf
                fi
        done
        sysctl --system || true
}

public::docker::install() {

    if [ "$CLOUD_TYPE" == "private" ];then
        echo "add tianji root ca ."
        # try add root ca
        public::cert::addCA
    fi

        set +e
        docker version >/dev/null 2>&1
        i=$?
        set -e
        v=$(docker version | grep Version | awk '{gsub(/-/, ".");print $2}' | uniq)
        if [ $i -eq 0 ]; then
                if [[ "$DOCKER_VERSION" == "$v" ]]; then
                        public::common::log "docker has been installed , return. $DOCKER_VERSION"
                        return
                fi
        fi
        public::common::prepare_package "docker" $DOCKER_VERSION
        if [ "$OS" == "CentOS" ] || [ "$OS" == "RedHat" ] || [ "$OS" == "AliOS" ]; then
                if [ "$(rpm -qa docker-engine-selinux | wc -l)" == "1" ]; then
                        yum erase -y docker-engine-selinux
                fi
                if [ "$(rpm -qa docker-engine | wc -l)" == "1" ]; then
                        yum erase -y docker-engine
                fi
                if [ "$(rpm -qa docker-ce | wc -l)" == "1" ]; then
                        yum erase -y docker-ce
                fi
                if [ "$(rpm -qa container-selinux | wc -l)" == "1" ]; then
                        yum erase -y container-selinux
                fi

                if [ "$(rpm -qa docker-ee | wc -l)" == "1" ]; then
                        yum erase -y docker-ee
                fi

                local pkg=pkg/docker/$DOCKER_VERSION/rpm/
                rpm --replacepkgs -Uvh --nodeps --force `ls $pkg |xargs -I '{}' echo -n "$pkg{} "`
        elif [ "$OS" == "Ubuntu" ]; then
                if [ "$need_reinstall" == "true" ]; then
                        if [ "$(echo $v | grep ee | wc -l)" == "1" ]; then
                                apt purge -y docker-ee docker-ee-selinux
                        elif [ "$(echo $v | grep ce | wc -l)" == "1" ]; then
                                apt purge -y docker-ce docker-ce-selinux container-selinux
                        else
                                apt purge -y docker-engine
                        fi
                fi
                dir=pkg/docker/$DOCKER_VERSION/debain
                dpkg -i $(ls $dir | xargs -I '{}' echo -n "$dir/{} ")
        elif [ "$OS" == "SUSE" ]; then
                if [ "$(rpm -qa docker-engine-selinux | wc -l)" == "1" ]; then
                        zypper rm -y docker-engine-selinux
                fi
                if [ "$(rpm -qa docker-engine | wc -l)" == "1" ]; then
                        zypper rm -y docker-engine
                fi
                if [ "$(rpm -qa docker-ce | wc -l)" == "1" ]; then
                        zypper rm -y docker-ce
                fi
                if [ "$(rpm -qa container-selinux | wc -l)" == "1" ]; then
                        zypper rm -y container-selinux
                fi

                if [ "$(rpm -qa docker-ee | wc -l)" == "1" ]; then
                        zypper rm -y docker-ee
                fi
                local pkg=pkg/docker/$KUBE_VERSION/rpm/
                zypper --no-gpg-checks install -y $(ls $pkg | xargs -I '{}' echo -n "$pkg{} ")
        else
                public::common::log "install docker with [unsupported OS version] error!"
                exit 1
        fi
        public::docker::config
}

public::docker::daemon_config_json() {
        if public::common::region4mirror "$REGION"; then
                local mirror=""https://pqbap4ya.mirror.aliyuncs.com""
        fi

        if [ "$DOCKER_VERSION" != "17.06.2.ce" ]; then
                local use_systemd_cgroup='native.cgroupdriver=systemd'
        fi

        if [ ! -d /etc/docker/ ]; then
                mkdir -p /etc/docker/
        fi

        if [ "1" == "$GPU_FOUNDED" ]; then
                cat >/etc/docker/daemon.json <<-EOF
{
    "default-runtime": "nvidia",
    "runtimes": {
        "nvidia": {
            "path": "/usr/bin/nvidia-container-runtime",
            "runtimeArgs": []
        }
    },
    "exec-opts": ["$use_systemd_cgroup"],
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m",
        "max-file": "10"
    },
    "bip": "169.254.123.1/24",
    "oom-score-adjust": -1000,
    "registry-mirrors": [$mirror],
    "insecure-registries" : ["`echo $KUBE_REPO_PREFIX|awk -F "/" '{print $1}'`","`echo $INSECURE_REGISTRY`", "`echo $INSECURE_REGISTRY|awk -F ":" '{print $1}'`"],
    "storage-driver": "overlay2",
    "storage-opts":["overlay2.override_kernel_check=true"],
    "live-restore": true
}
EOF
        else
                cat >/etc/docker/daemon.json <<-EOF
{
    "exec-opts": ["$use_systemd_cgroup"],
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m",
        "max-file": "10"
    },
    "bip": "169.254.123.1/24",
    "oom-score-adjust": -1000,
    "registry-mirrors": [$mirror],
    "insecure-registries" : ["`echo $KUBE_REPO_PREFIX|awk -F "/" '{print $1}'`","`echo $INSECURE_REGISTRY`", "`echo $INSECURE_REGISTRY|awk -F ":" '{print $1}'`"],
    "storage-driver": "overlay2",
    "storage-opts":["overlay2.override_kernel_check=true"],
    "live-restore": true
}
EOF
        fi
}

public::docker::config() {
        iptables -P FORWARD ACCEPT
        if [ ! -f /etc/systemd/system/docker.service ]; then
                cp /lib/systemd/system/docker.service /etc/systemd/system/docker.service
        fi
        if [ "$OS" == "CentOS" ] || [ "$OS" == "RedHat" ] || [ "$OS" == "AliOS" ]; then
                sed -i -e 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
                sed -i 's#LimitNOFILE=infinity#LimitNOFILE=1048576#g' /etc/systemd/system/docker.service
                if systemctl cat containerd &> /dev/null; then
                        sed -i 's#^After=network-online.target firewalld.service$#After=network-online.target firewalld.service containerd.service#g' 
                        /etc/systemd/system/docker.service
                fi
        fi

        public::docker::daemon_config_json

        sed -i "/ExecStart=/aExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT" /etc/systemd/system/docker.service

        systemctl daemon-reload
        systemctl enable docker.service
        # workaround for https://github.com/docker/for-linux/issues/556
        if systemctl cat containerd &> /dev/null; then
                systemctl start containerd
                retry 3 ctr --timeout 1s --connect-timeout 1s version
        fi
        systemctl restart docker.service
}

public::common::region4mirror() {
        local region=$1

        # set mirror for false, if the current region is not China Mainland
        if [[ $region == "cn-hongkong" ]] ||
                [[ $region != *"cn-"* ]]; then
                false
        else
                true
        fi
}

public::main::node_up()
{
    public::main::clean_cni

        public::common::nodeid

        public::common::install_package

        if [ $AUTO_FDISK != "" ]; then
            if [ "$DISK_DEVICE" == "" ];then
                # use default device as before
                DISK_DEVICE="/dev/xvdb"
            if [ -b "/dev/vdb" ]; then
                DISK_DEVICE="/dev/vdb"
            fi
                echo "disk device must be specified for auto mount docker disk. --disk-device=/dev/vdb"
            fi
                public::main::mount_disk "$DISK_DEVICE"
        fi

        if [[ $KUBE_VERSION == *"1.8"* ]]; then
                echo "Not support GPU yet for $KUBE_VERSION!"
        else
                public::main::nvidia_gpu
        fi

        if [ $FROM_ESS != "" ]; then
                public::main::attach_label
        fi

        discover="--discovery-token-unsafe-skip-ca-verification"

        kubeadm join --node-name "$NODE_ID" --token $TOKEN $APISERVER_LB $discover

        retry 30 grep cluster /etc/kubernetes/kubelet.conf
        cp /etc/kubernetes/kubelet.conf /etc/kubernetes/kubelet.old

        if [[ "$APISERVER_LB" != "1.1.1.1" ]]; then
                if [[ -f /etc/kubernetes/kubelet.conf ]]; then
                        sed -i "s#server: https://.*$#server: https://$APISERVER_LB#g" /etc/kubernetes/kubelet.conf
                fi
        fi

        systemctl restart kubelet
}

public::main::clean_cni()
{
    addr=$(ip addr show cni0|grep inet|awk '{print $2}')

    # We need to clean cni0 settings in case of conflict.
    if [ ! -z "$addr" ]; then
        set +e
        kubeadm reset
        ip addr del dev cni0 $addr
        rm -rf /var/lib/cni/*
        docker ps |grep k8s_|awk '{print $1}'|xargs -I '{}' docker stop {}
        set -e
    fi
}

public::main::attach_label() {
        KUBELET_EXTRA_ARGS=""

        if [ $TAINTS != "" ]; then
                TAINT_VARS="--register-with-taints=$TAINTS"
                KUBELET_EXTRA_ARGS="$KUBELET_EXTRA_ARGS $TAINT_VARS"
        fi

        if [ $LABELS != "" ]; then

                LABEL_VARS="--node-labels=$LABELS"
                KUBELET_EXTRA_ARGS="$KUBELET_EXTRA_ARGS $LABEL_VARS"

                if [ "1" == "$GPU_FOUNDED" ]; then
                        sed -i "s#^Environment="KUBELET_EXTRA_ARGS=--node-labels=#Environment="KUBELET_EXTRA_ARGS=$KUBELET_EXTRA_ARGS,#" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
                else
                        #sed -i '/^ExecStart=$/iEnvironment="KUBELET_EXTRA_ARGS=--feature-gates=DevicePlugins=true"' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
                        sed -i "/^ExecStart=$/iEnvironment="KUBELET_EXTRA_ARGS=$KUBELET_EXTRA_ARGS"" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
                fi
                systemctl daemon-reload
                systemctl restart kubelet

        else
                echo "No extra labels for this node"
        fi
}

public::main::mount_disk() {
        #config /etc/fstab and mount device
        device=$1
        if [ ! -b "$device" ]; then
                echo "mount_disk fail: ${device} is not a block dev"; exit 2
        fi
        set +e
        cat /etc/fstab | grep $device >/dev/null 2>&1
        i=$?
        set -e
        if [ $i -eq 0 ]; then
                echo "disk ${device} is already mounted"; return
        fi

    dev_slave="${device}1"
    if ! tune2fs -l "${dev_slave}" ; then
        echo "Formatting '${dev_slave}'"
        fdisk -S 56 "${device}" <<-EOF
                n
                p
                1


                wq
        EOF
        sleep 5
        mkfs.ext4 -i 8192 "${dev_slave}"
    fi

        docker_flag=0
        if [ -d "/var/lib/docker" ]; then
                docker_flag=1
                service docker stop
                rm -fr /var/lib/docker
        fi
        mkdir /var/lib/docker

        kubelet_flag=0
        if [ -d "/var/lib/kubelet" ]; then
                kubelet_flag=1
                service kubelet stop
                rm -fr /var/lib/kubelet
        fi
        mkdir /var/lib/kubelet

        if [ ! -d /var/lib/container ]; then
                mkdir /var/lib/container/
        fi

        mount -t ext4 ${dev_slave} /var/lib/container/
        mkdir /var/lib/container/kubelet /var/lib/container/docker
        echo "${dev_slave}    /var/lib/container/     ext4    defaults        0 0
/var/lib/container/kubelet /var/lib/kubelet none defaults,bind 0 0
/var/lib/container/docker /var/lib/docker none defaults,bind 0 0" >>/etc/fstab

        mount -a; df -h

        if [ $docker_flag == 1 ]; then
                service docker start
        fi
        if [ $kubelet_flag == 1 ]; then
                service kubelet start
        fi
}

public::main::nvidia_gpu() {
        current_region=$(curl --retry 5 -sSL http://100.100.100.200/latest/meta-data/region-id)
        if [ "" == "$current_region" ]; then
                echo "can not get regionid and instanceid! 
            curl --retry 5 -sSL http://100.100.100.200/latest/meta-data/region-id" && exit 256
        fi
        kube::main::detect_gpu
        if [ "1" == "$GPU_FOUNDED" ]; then
                nvidiaDriverInstallUrl=$PKG_FILE_SERVER/$CLOUD_TYPE/pkg/run/attach/$KUBE_VERSION/nvidia-gpu.sh

                curl --retry 5 -sSL ${nvidiaDriverInstallUrl} -o nvidia-gpu.sh
                chmod u+x nvidia-gpu.sh
                source nvidia-gpu.sh
                public::nvidia::enable_gpu_capability
        else
                echo "Do nothing for non-GPU node"
        fi
        # public::nvidia::deploy_static_pod
}

public::main::deploy_nvidia_plugin() {
        set +e
        kube::main::detect_gpu
        if [ "1" == "$GPU_FOUNDED" ]; then
                public::nvidia::deploy_static_pod
        else
                echo "Do nothing for non-GPU node"
        fi
        set -e
}

kube::main::detect_gpu() {
        set +e
        kube::main::setup_lspci $OS
        lspci | grep -i nvidia >/dev/null 2>&1
        i=$?
        if [ $i -eq 0 ]; then
                export GPU_FOUNDED=1
        fi
        set -e
}

kube::main::setup_lspci() {
        which lspci >/dev/null 2>&1
        i=$?
        if [ $i -ne 0 ]; then
                if [ "$1" == "CentOS" ]; then
                        yum install -y pciutils
                else
                        apt install -y pciutils
                fi
        fi
}

public::common::get_node_info() {
        if [ "$CLOUD_TYPE" == "public" -o "$CLOUD_TYPE" == "private" ]; then
                insid=$(curl --retry 5 -sSL http://100.100.100.200/latest/meta-data/instance-id)
                info=$(curl --retry 5 -H "Date:$(date -R)" -sfSL "$openapi/token/${OPENAPI_TOKEN}/instance/${insid}/node_info" | grep 'w')
                eval "$info"
                export DOCKER_VERSION=$docker_version
                export CLUSTER_DNS=$cluster_dns
                export TOKEN=$token
                export APISERVER_LB=$endpoint
        fi
}

public::common::callback()
{
    if [ "$CLOUD_TYPE" != "agility" ];then
        if [ "$CLOUD_TYPE" == "private" ];then
            callback_url="${callback_url/cs-anony.aliyuncs.com/cs-intranet.inter.env17e.shuguang.com}"
        fi
        curl -H "Date:`date -R`" -X POST -sfSLk "${callback_url}"
        echo "======================================="
        echo "                 SUCCESS               "
        echo "======================================="
    fi
}

public::main::cleanup() {
        addr=$(ip addr show cni0 | grep inet | awk '{print $2}')

        # We need to clean cni0 settings in case of conflict.
        if [ ! -z "$addr" ]; then
                set +e
                ip addr del dev cni0 $addr
                rm -rf /var/lib/cni/*
                docker ps | grep k8s_ | awk '{print $1}' | xargs -I '{}' docker stop {}
                set -e
        fi

                rm -rf /etc/kubernetes

                set +e
                which kubelet >/dev/null 2>&1
                i=$?
                if [ $i -eq 0 ]; then
                        systemctl stop kubelet
                fi
                set -e
}

main() {
        while
                [[ $# -gt 0 ]]
        do
                key="$1"

                case $key in
                --docker-version)
                        export DOCKER_VERSION=$2
                        shift
                        ;;
                --cluster-dns)
                        export CLUSTER_DNS=$2
                        shift
                        ;;
                --token)
                        export TOKEN=$2
                        shift
                        ;;
                --endpoint)
                        export APISERVER_LB=$2
                        shift
                        ;;
                --openapi-token)
                        export OPENAPI_TOKEN=$2
                        shift
                        ;;
                --ess)
                        export FROM_ESS=$2
                        export AUTO_FDISK="true"
                        shift
                        ;;
                --auto-fdisk)
                        export AUTO_FDISK="true"
                        ;;

                --disk-device)
                        export DISK_DEVICE=$2
                        shift
                        ;;
                --labels)
                        export LABELS=$2
                        shift
                        ;;
                --taints)
                        export TAINTS=$2
                        shift
                        ;;
                *)
                        public::common::log "unknown option [$key]"
                        #exit 1
                        ;;
                esac
                shift
        done

        if [ "$OPENAPI_TOKEN" != "" ]; then
                # Using default cidr.
                public::common::get_node_info
        fi

        if [ "$DOCKER_VERSION" == "" ]; then
                # Using default cidr.
                public::common::log "DOCKER_VERSION $DOCKER_VERSION is not set."
                exit 1
        fi

        KUBE_VERSION=$(curl -k https://$APISERVER_LB/version | grep gitVersion | awk '{print $2}' | cut -f2 -d ")
        export KUBE_VERSION=${KUBE_VERSION:1}
        if [ "$KUBE_VERSION" == "" ]; then
                # Using default cidr.
                public::common::log "KUBE_VERSION $KUBE_VERSION is failed to set."
                exit 1
        fi

        if [ "$TOKEN" == "" ]; then
                # Using default cidr.
                public::common::log "TOKEN $TOKEN is not set."
                exit 1
        fi

        if [ "$CLUSTER_DNS" == "" ]; then
                # Using default cidr.
                public::common::log "CLUSTER_DNS $CLUSTER_DNS is not set."
                exit 1
        fi

        public::common::nodeid
        public::common::os_env
        public::common::region

        # Do some clean up
        public::main::cleanup

        # 首先从本地读取相应版本的tar包。当所需要的安装包不存在的时候
        # 如果设置了参数PKG_FILE_SERVER，就从该Server上下载。
        # 如果是在公有云上执行，可以使用内网oss地址
        if [ "$PKG_FILE_SERVER" == "" ]; then
                export PKG_FILE_SERVER=http://aliacs-k8s-$REGION.oss-$REGION-internal.aliyuncs.com
        fi

        # 安装Kubernetes时候会启动一些AddOn插件的镜像。
        # 改插件设置镜像仓库的前缀。
        if [ "$KUBE_REPO_PREFIX" == "" ]; then
                export KUBE_REPO_PREFIX=registry-vpc.$REGION.aliyuncs.com/acs
        fi

        public::main::node_up
        public::main::deploy_nvidia_plugin

        if [ "$OPENAPI_TOKEN" != "" ]; then
                public::common::callback
        fi

}

main "$@"