zoukankan      html  css  js  c++  java
  • 企业级高可用Harbor 2.3

    环境说明

    主机名 IP 系统 用途
    harbor01.ecloud.com 192.168.32.137 CentOS 7.6 安装harbor仓库、HTTPS
    harbor02.ecloud.com 192.168.32.145 CentOS 7.6 安装harbor仓库、HTTPS
    db.ecloud.com 192.168.32.147 CentOS 7.6 安装nfs、redis、postgresql

    两台harbor仓库使用NFS文件系统来存储数据,使用外部的redis和postgresql来实现高可用。

    安装NFS

    停止防火墙

    systemctl stop firewalld && systemctl disable firewalld
    

    下载nfs-server包

    yum install -y nfs-utils
    

    配置nfs

    mkdir -p /nfs/harbor
    echo '/nfs/harbor 192.168.32.0/24(rw,no_root_squash)' >> /etc/exports
    

    启动nfs

    systemctl start nfs.service && systemctl enable nfs.service
    

    安装redis

    设置环境变量

    echo 'net.core.somaxconn=511' >> /etc/sysctl.d/redis.conf
    echo 'vm.overcommit_memory=1' >> /etc/sysctl.d/redis.conf
    sysctl -p /etc/sysctl.d/redis.conf
    
    ulimit -n 65535
    echo '*    hard    nofile    65536' >> /etc/security/limits.d/redis.conf
    echo '*    soft    nofile    65536' >> /etc/security/limits.d/redis.conf
    

    下载依赖包

    yum install -y wget gcc bzip2 readline-devel zlib-devel
    

    下载redis包

    wget https://download.redis.io/releases/redis-6.2.4.tar.gz
    

    解压包

    mkdir -p /app/redis
    tar xf redis-6.2.4.tar.gz -C /app/redis
    

    创建启动用户

    adduser redis && echo redis | passwd redis --stdin
    chown -R redis.redis /app/redis
    su - redis
    

    编译redis

    cd /app/redis/redis-6.2.4
    make -j8
    

    修改配置文件

    sed -ri 's/port 6379/port 7777/g' redis.conf
    sed -ri 's/# requirepass.*/requirepass redis/g' redis.conf
    sed -ri 's@(^logfile).*@1 "/app/redis/redis-6.2.4/redis.log"@g' /app/redis/redis-6.2.4/redis.conf
    sed -ri 's/^(bind).*/1 * -::*/g' /app/redis/redis-6.2.4/redis.conf
    

    启动redis

    /app/redis/redis-6.2.4/src/redis-server /app/redis/redis-6.2.4/redis.conf &
    

    测试redis

    /app/redis/redis-6.2.4/src/redis-cli -h 192.168.32.147 -p 7777
    192.168.32.147:7777> auth redis  # redis是验证的密码,即上面设置requirepass的值
    

    输入 auth redis 返回是 ok 就正常。

    安装postgresql

    下载依赖包

    yum install -y wget gcc bzip2 readline-devel zlib-devel
    

    下载postgresql包

    wget https://mirrors.tuna.tsinghua.edu.cn/postgresql/source/v12.7/postgresql-12.7.tar.bz2
    

    解压postgresql包

    tar xf postgresql-12.7.tar.bz2
    cd postgresql-12.7
    

    编译postgresql包

    mkdir -p /app/pgsql
    ./configure --prefix=/app/pgsql
    make -j8 world -j8
    make -j8 install-world
    

    添加启动用户

    adduser postgres && echo postgres | passwd postgres --stdin
    

    修改目录权限

    mkdir /app/pgsql/{data,logs}
    chown -R postgres:postgres /app/pgsql
    su - postgres
    

    初始化数据库

    /app/pgsql/bin/initdb -D /app/pgsql/data -U admin -W -A md5
    

    修改配置文件

    sed -ri 's/^#(port).*/1 = 5400/g' /app/pgsql/data/postgresql.conf
    sed -ri 's/^#(log_destination.*)/1/g' /app/pgsql/data/postgresql.conf
    sed -ri 's/^#(logging_collector).*(#.*)/1 = no 2/g' /app/pgsql/data/postgresql.conf
    sed -ri "s@^#(log_directory).*(#.*)@1 = '/app/postgresql/logs/' 2@g" /app/pgsql/data/postgresql.conf
    sed -ri 's/^#(log_filename.*)/1/g' /app/pgsql/data/postgresql.conf
    sed -ri 's/^#(log_file_mode.*)/1/g' /app/pgsql/data/postgresql.conf
    sed -ri "s@^#(listen_addresses).*(#.*)@1 = '*' 2@g" /app/pgsql/data/postgresql.conf
    

    修改登录权限

    vi /app/pgsql/data/pg_hba.conf
    # IPv4 local connections:
    host    all             all             127.0.0.1/32            md5
    host    all             all             192.168.32.0/24         md5		# 新增的一行
    

    在IPv4连接下面添加一个行,整个网段可以通过账号密码登录。

    启动postgresql

    /app/pgsql/bin/pg_ctl -D /app/pgsql/data start
    

    获取habor数据

    安装harbor仓库

    获取数据,其中一台harbor主机执行即可

    wget https://github.com/goharbor/harbor/releases/download/v2.3.0/harbor-offline-installer-v2.3.0.tgz
    tar xf harbor-offline-installer-v2.3.0.tgz
    cd harbor
    cp harbor.yml.tmpl harbor.yml
    vi harbor.yml
    # 1.修改名称
    hostname: harbor.ecloud.com
    # 2.注释https相关配置
    
    ./prepare
    ./install.sh
    

    导出postgresql数据

    docker exec -it harbor-db sh
    pg_dump registry > /tmp/registry.sql
    docker cp harbor-db:/tmp/registry.sql ~
    

    复制到db主机上

    scp ~/registry.sql postgres@192.168.32.147:~
    

    删除harbor仓库

    docker-compose down
    

    导入数据

    # 创建库
    /app/pgsql/bin/createdb -h localhost -p 5400 -U admin registry
    /app/pgsql/bin/createdb -h localhost -p 5400 -U admin notaryserver
    /app/pgsql/bin/createdb -h localhost -p 5400 -U admin notarysigner
    
    # 导入数据
    /app/pgsql/bin/psql -h localhost -p 5400 -U admin -d registry -f ~/registry.sql
    
    # 验证
    /app/pgsql/bin/psql -h localhost -p 5400 -U admin registry
    

    安装harbor仓库

    安装nfs客户端

    mkdir /data
    yum -y install nfs-utils
    echo "192.168.32.147:/nfs/harbor /data nfs defaults,_netdev 0 0">> /etc/fstab
    mount -a
    

    生成harbor服务之间的证书(单节点执行)

    docker pull goharbor/prepare:v2.3.0
    docker run -v /:/hostfs goharbor/prepare:v2.3.0 gencert -p /data/tls/harbor/
    

    一个客户端执行即可,NFS文件系统共享数据

    生成nginx证书(单节点执行)

    # 创建目录
    mkdir -p /data/tls/nginx && cd /data/tls/nginx
    
    # 生成 CA 证书私钥
    openssl genrsa -out ca.key 4096
    
    # 生成 CA 证书
    openssl req -x509 -new -nodes -sha512 -days 3650 
    -subj "/C=CN/ST=Guangdong/L=Guangzhou/O=Personal/OU=Personal/CN=ecloud.com" 
    -key ca.key 
    -out ca.crt
    
    # 生成私钥
    openssl genrsa -out ecloud.com.key 4096
    
    # 生成证书签名请求 (CSR)
    openssl req -sha512 -new 
    -subj "/C=CN/ST=Guangdong/L=Guangzhou/O=Personal/OU=Personal/CN=ecloud.com" 
    -key ecloud.com.key 
    -out ecloud.com.csr
    
    # 生成 x509 v3 扩展文件
    cat > v3.ext <<-EOF
    authorityKeyIdentifier=keyid,issuer
    basicConstraints=CA:FALSE
    keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
    extendedKeyUsage = serverAuth
    subjectAltName = @alt_names
    
    [alt_names]
    DNS.1=harbor.ecloud.com	# 写成harbor.yml中的hostname的值
    DNS.2=*.ecloud.com
    EOF
    
    # 使用该v3.ext文件为您的 Harbor 主机生成证书。
    openssl x509 -req -sha512 -days 3650 
        -extfile v3.ext 
        -CA ca.crt -CAkey ca.key -CAcreateserial 
        -in ecloud.com.csr 
        -out ecloud.com.crt
    
    # 转换yourdomain.com.crt为yourdomain.com.cert,供 Docker 使用。
    openssl x509 -inform PEM -in ecloud.com.crt -out ecloud.com.cert
    

    一个客户端执行即可,NFS文件系统共享数据

    docker配置证书

    rm -rf /etc/docker/certs.d/harbor.ecloud.com
    mkdir -p /etc/docker/certs.d/harbor.ecloud.com/
    cp /data/tls/nginx/{ecloud.com.cert,ecloud.com.key,ca.crt} /etc/docker/certs.d/harbor.ecloud.com/
    
    # 重启docker服务
    systemctl restart docker
    

    将服务器证书和密钥复制到 Harbor 主机上的 certficates 文件夹中,目录名称创建为harbor.yml中的hostname的值

    下载harbor包

    wget https://github.com/goharbor/harbor/releases/download/v2.3.0/harbor-offline-installer-v2.3.0.tgz
    

    解压harbor仓库

    tar xf harbor-offline-installer-v2.3.0.tgz
    

    修改harbor配置

    cd harbor
    cp harbor.yml.tmpl harbor.yml
    vim harbor.yml
    

    修改的内容如下:

    # 这个就是docker login 的名称
    hostname: harbor.ecloud.com
    # 配置nginx的证书
    https
      # https port for harbor, default is 443
      port: 443
      # The path of cert and key files for nginx
      certificate: /data/tls/nginx/ecloud.com.crt
      private_key: /data/tls/nginx/ecloud.com.key
    # 配置harbor服务之间的证书
    internal_tls:
      # set enabled to true means internal tls is enabled
      enabled: true
      # put your cert and key files on dir
      dir: /data/tls/harbor
    # 启用外部代理
    external_url: https://harbor.ecloud.com
    # 连接外部数据库
    external_database:
      harbor:
        host: 192.168.32.147
        port: 5400
        db_name: registry
        username: admin
        password: admin
        ssl_mode: disable
        max_idle_conns: 2
        max_open_conns: 0
      notary_signer:
        host: 192.168.32.147
        port: 5400
        db_name: notarysigner
        username: admin
        password: admin
        ssl_mode: disable
      notary_server:
        host: 192.168.32.147
        port: 5400
        db_name: notaryserver
        username: admin
        password: admin
        ssl_mode: disable
    # 连接外部redis
    external_redis:
      # support redis, redis+sentinel
      # host for redis: <host_redis>:<port_redis>
      # host for redis+sentinel:
      #  <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
      host: 192.168.32.147:7777
      password: redis
      # sentinel_master_set must be set to support redis+sentinel
      #sentinel_master_set:
      # db_index 0 is for core, it's unchangeable
      registry_db_index: 1
      jobservice_db_index: 2
      chartmuseum_db_index: 3
      trivy_db_index: 5
      idle_timeout_seconds: 30
    

    安装harbor

    ./prepare
    ./install.sh
    

    安装harbor扩展功能(char、漏洞扫描、镜像签名)

    下载离线的trivy包

    wget https://github.com/aquasecurity/trivy-db/releases/download/v1-2021072600/trivy-light-offline.db.tgz
    

    下载离线的trivy包的话,一定要可以tag改成今天的 v1-2021072600 ,否则提示 ERROR 404: Not Found.

    解压trivy

    mkdir -p /data/trivy-adapter/trivy/db/
    tar xf trivy-offline.db.tgz -C /data/trivy-adapter/trivy/db/
    chown 10000:10000 -R /data/trivy-adapter/trivy/db
    

    安装扩展功能

    ./prepare --with-notary --with-chartmuseum --with-trivy
    docker-compose -f docker-compose.yml up -d
    

    如果只安装某个服务的话,那就添加对应的选项。

    配置notary

    # 配置环境变量
    cat > /etc/profile.d/notary << EOF
    export DOCKER_CONTENT_TRUST=1
    export DOCKER_CONTENT_TRUST_SERVER=https://harbor.ecloud.com:4443
    export NOTARY_SNAPSHOT_PASSPHRASE=Harbor12345
    export NOTARY_ROOT_PASSPHRASE=Harbor12345
    export NOTARY_TARGETS_PASSPHRASE=Harbor12345
    EOF
    source /etc/profile.d/notary
    
    # docker证书
    mkdir -p ~/.docker/tls/harbor.ecloud.com:4443
    cp /etc/docker/certs.d/harbor.ecloud.com/ca.crt  ~/.docker/tls/harbor.ecloud.com:4443
    
    # 安装notary命令
    wget https://github.com/theupdateframework/notary/releases/download/v0.6.1/notary-Linux-amd64
    chmod +x notary-Linux-amd64 && mv notary-Linux-amd64 /usr/local/bin/notary
    
    # 初始化集合
    notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt init harbor.ecloud.com/library/nginx
    
    # 查看拥有的key信息
    notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt key list
    
    # tag签名
    echo 123456 > file
    notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt add harbor.ecloud.com/library/nginx latest file
    
    # 查看该信任集合的状态
    notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt status harbor.ecloud.com/library/nginx
    
    # 该信任集合的改变推送至notary服务端
    notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt publish harbor.ecloud.com/library/nginx
    

    测试

    下载nginx镜像

    export DOCKER_CONTENT_TRUST=0
    docker pull nginx
    

    修改tag

    docker tag nginx:latest harbor.ecloud.com/library/nginx:latest 
    

    登录harbor

    echo 'Harbor12345' | docker login harbor.ecloud.com -u admin --password-stdin
    

    推镜像

    docker push harbor.ecloud.com/library/nginx:latest
    The push refers to repository [harbor.ecloud.com/library/nginx]
    e3135447ca3e: Pushed 
    b85734705991: Pushed 
    988d9a3509bb: Pushed 
    59b01b87c9e7: Pushed 
    7c0b223167b9: Pushed 
    814bff734324: Pushed 
    latest: digest: sha256:3f13b4376446cf92b0cb9a5c46ba75d57c41f627c4edb8b635fa47386ea29e20 size: 1570
    Signing and pushing trust metadata
    Enter passphrase for repository key with ID cfa46c2: 
    Enter passphrase for repository key with ID d6efb6a: 
    Successfully signed harbor.ecloud.com/library/nginx:latest
    

    验证

    image-20210726180132713

    image-20210726180239816

    image-20210726180303239

    image-20210726180729925

    keepalived配置

    关闭防火墙及selinux

    systemctl stop firewalld 
    setenforce 0
    

    创建ARP冲突检测脚本

    vi /etc/init.d/realserver
    
    #!/bin/bash
    # description:Script to start LVS DR real server.
    . /etc/rc.d/init.d/functions
    VIP=192.168.32.100
    
    #修改相应的VIP
    case "$1" in
        start)
            #启动 LVS-DR 模式,real server on this machine. 关闭ARP冲突检测。
            echo "Start LVS of Real Server!"
            /sbin/ifconfig lo down
            /sbin/ifconfig lo up
            echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
            echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
            echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
            echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
            /sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
            /sbin/route add -host $VIP dev lo:0
            sudo sysctl -p
        ;;
    
        stop)
            #停止LVS-DR real server loopback device(s).
            echo "Close LVS Director Server!"
            /sbin/ifconfig lo:0 down
            echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
            echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
            echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
            echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
            sudo sysctl -p
        ;;
        
        status)
            # Status of LVS-DR real server.
            islothere=`/sbin/ifconfig lo:0 | grep $VIP`
            isrothere=`netstat -rn | grep "lo:0" | grep $VIP`
            if [ ! "$islothere" -o ! "isrothere" ];then
            # Either the route or the lo:0 device
            # not found.
                echo "LVS-DR real server Stopped!"
            else
                echo "LVS-DR real server Running..."
            fi
        ;;
        
        *)
            # Invalid entry.
            echo "$0: Usage: $0 {start|status|stop}"
            exit 1
        ;;
    
    esac
    

    关闭ARP冲突检测

    chmod +x /etc/init.d/realserver 
    chmod +x /etc/rc.d/init.d/functions
    service realserver start
    

    下载安装包

    yum install -y keepalived ipvsadm
    

    修改keepalive master配置

    mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
    vi /etc/keepalived/keepalived.conf
    
    global_defs {
       smtp_server 127.0.0.1
    }
    
    vrrp_instance VI_1 {
        state MASTER          #指定Keepalived的角色,MASTER为主,BACKUP为备 记得大写
        interface ens33       #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
        virtual_router_id 51  #虚拟路由编号,主备要一致
        priority 100          #定义优先级,数字越大,优先级越高,主DR必须大于备用DR
        advert_int 1          #检查间隔,默认为1s
        authentication {      #这里配置的密码最多为8位,主备要一致,否则无法正常通讯
            auth_type PASS
            auth_pass 1111
        }
        virtual_ipaddress {
            192.168.32.100    #定义虚拟IP(VIP)为192.168.1.200,可多设,每行一个
        }
    }
    
    # 定义对外提供服务的LVS的VIP以及port
    virtual_server 192.168.32.100 443 {
        delay_loop 6          # 设置健康检查时间,单位是秒
        lb_algo wlc           # 设置负载调度的算法为wlc
        lb_kind DR            # 设置LVS实现负载的机制,有NAT、TUN、DR三个模式
        nat_mask 255.255.255.0
        persistence_timeout 0
        protocol TCP
        real_server 192.168.32.137 443 {  # 指定real server1的IP地址
            weight 3   # 配置节点权值,数字越大权重越高
            TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 443
            }
        }
        real_server 192.168.32.145 443 {  # 指定real server2的IP地址
            weight 3  # 配置节点权值,数字越大权重越高
            TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 443
            }
         }
    }
    

    修改keepalived backup配置

    mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
    vi /etc/keepalived/keepalived.conf
    
    global_defs {
       smtp_server 127.0.0.1
    }
    
    vrrp_instance VI_1 {
        state BACKUP #指定Keepalived的角色,MASTER为主,BACKUP为备 记得大写
        interface ens33  #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
        virtual_router_id 51  #虚拟路由编号,主备要一致
        priority 50  #定义优先级,数字越大,优先级越高,主DR必须大于备用DR
        advert_int 1  #检查间隔,默认为1s
        authentication {   #这里配置的密码最多为8位,主备要一致,否则无法正常通讯
            auth_type PASS
            auth_pass 1111
        }
        virtual_ipaddress {
            192.168.32.100  #定义虚拟IP(VIP)为192.168.1.200,可多设,每行一个
        }
    }
    
    # 定义对外提供服务的LVS的VIP以及port
    virtual_server 192.168.32.100 443 {
        delay_loop 6 # 设置健康检查时间,单位是秒
        lb_algo wlc  # 设置负载调度的算法为wlc
        lb_kind DR   # 设置LVS实现负载的机制,有NAT、TUN、DR三个模式
        nat_mask 255.255.255.0
        persistence_timeout 0
        protocol TCP
        real_server 192.168.32.137 443 {  # 指定real server1的IP地址
            weight 3   # 配置节点权值,数字越大权重越高
            TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 443
            }
        }
    
        real_server 192.168.32.145 443 {  # 指定real server2的IP地址
            weight 3  # 配置节点权值,数字越大权重越高
            TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 443
            }
         }
    }
    

    启动服务

    systemctl start keepalived
    

    docker客户端的配置

    获取harbor仓库的证书

    scp -r 192.168.32.137:/etc/docker/certs.d /etc/docker/
    systemctl restart docker
    

    添加完证书后,需要重启 docker 服务。

    IP域名映射

    echo 192.168.32.100 harbor.ecloud.com >> /etc/hosts
    

    192.168.32.100 是VIP地址。

    所有配置已设置,可以体验私有的仓库

  • 相关阅读:
    解决SSH连接Linux超时自动断开
    小程序选项卡
    vue 封装axios 请求 统一管理方法1
    vue 中使用echar
    vue element 做表格分页
    vue echar使用
    旋转
    vue 登录切换页面
    vue 根据输入的身份号码,自动获取年龄
    vue 手机号码验证 。点击获取验证码
  • 原文地址:https://www.cnblogs.com/mycloudedu/p/15066528.html
Copyright © 2011-2022 走看看