zoukankan      html  css  js  c++  java
  • CentOS7下搭建FastDfs(V5.11)+Keepalived分布式集群部署

    FastDfs介绍

    http://kb.cnblogs.com/page/82280/

    1、准备


    系统

    CentOS7 最小化安装。

    #VIP虚拟IP 
    10.1.6.218
    #Keepalived
    10.1.6.216  #Keepalived1+Nginx
    10.1.6.217  #Keepalived2+Nginx
    #Tracke
    10.1.6.210  #Tracke1+Nginx
    10.1.6.211  #Tracke2+Nginx
    #Group1
    10.1.6.212  #Storage1+Nginx
    10.1.6.213  #Storage2+Nginx
    #Group2
    10.1.6.214  #Storage3+Nginx
    10.1.6.215  #Storage4+Nginx
    #部署结构示意
                           +-----+
                           | VIP |
                           +-----+
                              |
            +------------+    |    +------------+
            | Keepalived |----+----| Keepalived |
            | Nginx      |         | Nginx      |
            +------------+         +------------+
                    |←-----------------→|
                    |                   |
                    ↓                   ↓
               +---------+         +---------+
               | Tracke1 |         | Tracke2 |
               | Nginx   |         | Nginx   |
               +---------+         +---------+
                    |←-----------------→|
                    |                   |
                    ↓                   ↓
     +-------------------+         +-------------------+
     | Group1            |         | Group2            |
     | Storage+Nginx     |         | Storage+Nginx     |
     | +----+     +----+ |         | +----+     +----+ |
     | | S1 |--+--| S2 | |         | | S3 |--+--| S4 | |
     | | Ng |     | Ng | |         | | Ng |     | Ng | |
     | +----+     +----+ |         | +----+     +----+ |
     +-------------------+         +-------------------+
    
    #测试环境,禁用防火墙
    systemctl stop firewalld.service
    systemctl disable firewalld.service
    
    下载
    文件 下载地址 介绍
    FastDFS https://github.com/happyfish100/fastdfs (Version 5.11 2017-05-26 fastdfs-master.zip)
    libfastcommon https://github.com/happyfish100/libfastcommon FastDFS 运行所需要的基础库(Version 1.36 2017-03-08 libfastcommon-master.zip)
    fastdfs-nginx-module https://github.com/happyfish100/fastdfs-nginx-module storage节点http服务nginx模块(Version 1.20 2017-05-26 fastdfs-nginx-module-master.zip)
    Nginx http://nginx.org/ (nginx-1.12.0)
    ngx_cache_purge https://github.com/FRiCKLE/ngx_cache_purge/tree/2.3 Nginx图片缓存清除模块

    将文件下载到 /usr/local/src 文件夹。

    2、安装


    依赖安装
    #安装 gcc、libevent、perl
    yum -y install gcc-c++
    yum -y install libevent
    yum -y install perl
    #安装zip解压
    yum -y install unzip
    
    libfastcommon安装
    #libfastcommon解压、安装
    unzip libfastcommon-master.zip
    ln -s libfastcommon-master libfastcommon
    cd libfastcommon
    ./make.sh
    ./make.sh install
    

    安装完成后/usr/lib64、/usr/lib文件夹内会有libfastcommon.so、libfdfsclient.so文件。

    #V5.11可忽略
    #FastDFS主程序设置的lib目录是/usr/local/lib,所以此处需要重新设置软链接
    ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
    ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so
    ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so
    ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so
    
    fastdfs安装
    #fastdfs解压、安装
    unzip fastdfs-master.zip
    cd fastdfs-master
    ./make.sh
    ./make.sh install
    

    安装完成后/etc/fdfs文件夹内会有client.conf.sample、storage.conf.sample、storage_ids.conf.sample、tracker.conf.sample文件。

    #将示例配置文件拷贝一份
    cd /etc/fdfs/
    cp client.conf.sample client.conf
    cp storage.conf.sample storage.conf
    cp tracker.conf.sample tracker.conf
    

    3、启动服务、测试


    #创建存储数据、日志目录
    mkdir -p /usr/local/FastDFS/Tracker
    mkdir -p /usr/local/FastDFS/Storage
    mkdir -p /usr/local/FastDFS/FileData
    mkdir -p /usr/local/FastDFS/Client
    
    Tracker配置
    #修改tracker.conf配置
    vi tracker.conf
    #设置Tracker的数据文件和日志目录
    base_path=/usr/local/FastDFS/Tracker
    #0轮询策略方便测试,默认是2(负载均衡策略)
    store_lookup=0
    #默认
    port=22122
    http.server_port=8080
    #启动Tracker服务
    service fdfs_trackerd start
    ps -ef|grep fdfs
    netstat -unltp|grep fdfs
    

    Storage配置
    vi storage.conf
    #设置Storage的数据文件和日志目录
    base_path=/usr/local/FastDFS/Storage
    #实际文件存储路径
    store_path0=/usr/local/FastDFS/FileData
    #tracker 服务器的 IP地址和端口号
    tracker_server=10.1.6.210:22122
    tracker_server=10.1.6.211:22122
    #默认
    group_name=group1
    port=23000
    store_path_count=1
    #启动Storage服务
    service fdfs_storaged start
    ps -ef|grep fdfs
    netstat -unltp|grep fdfs
    

    #查看storage服务器是否已经登记到tracker服务器
    fdfs_monitor /etc/fdfs/storage.conf
    
    [root@localhost logs]# fdfs_monitor /etc/fdfs/storage.conf
    [2017-07-01 19:36:50] DEBUG - base_path=/usr/local/FastDFS/Storage, connect_timeout=30, network_timeout=60, tracker_server_count=2, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
    
    server_count=2, server_index=1
    
    tracker server is 10.1.6.211:22122
    
    group count: 2
    
    Group 1:
    group name = group1
    ...
    current trunk file id = 0
    
            Storage 1:
                    id = 10.1.6.212
                    ip_addr = 10.1.6.212  ACTIVE
                    http domain = 
                    version = 5.11
                    ...
                    last_synced_timestamp = 2017-07-01 19:36:50 (0s delay)
            Storage 2:
                    id = 10.1.6.213
                    ip_addr = 10.1.6.213  ACTIVE
                    http domain = 
                    version = 5.11
                    ...
                    last_synced_timestamp = 2017-07-01 19:36:50 (-1s delay)
    
    Group 2:
    group name = group2
    ...
    current trunk file id = 0
    
            Storage 1:
                    id = 10.1.6.214
                    ip_addr = 10.1.6.214  ACTIVE
                    http domain = 
                    version = 5.11
                    ...
                    last_synced_timestamp = 2017-07-01 19:36:50 (0s delay)
            Storage 2:
                    id = 10.1.6.215
                    ip_addr = 10.1.6.215 (localhost.localdomain)  ACTIVE
                    http domain = 
                    version = 5.11
                    ...
                    last_synced_timestamp = 2017-07-01 19:36:50 (0s delay)
    
    测试(上传、删除)
    #配置client.conf
    vi /etc/fdfs/client.conf
    base_path=/usr/local/FastDFS/Client
    #tracker服务器IP地址和端口号
    tracker_server=10.1.6.210:22122
    tracker_server=10.1.6.211:22122
    # tracker 服务器的 http 端口号,必须和tracker的设置对应起来
    http.tracker_server_port=8080
    #上传
    fdfs_upload_file /etc/fdfs/client.conf /usr/local/src/fastdfs-master/conf/anti-steal.jpg
    #删除
    fdfs_delete_file /etc/fdfs/client.conf group1/M00/00/00/CgEG5llJQGGA01_1AABdrzgsqUU484.jpg
    

    #在/usr/local/FastDFS/FileData/data/00/00下查看上传文件。
    [root@localhost 00]# pwd
    /usr/local/FastDFS/FileData/data/00/00
    [root@localhost 00]# ll
    总用量 24
    -rw-r--r--. 1 root root 23981 6月  20 23:33 CgEG5llJQGGAO1_1AABdrZgsqUU484.jpg
    

    4、Storage存储节点(10.1.6.212、10.1.6.213、10.1.6.214、10.1.6.215)安装Nginx和fastdfs-nginx-module模块


    #安装nginx需要的一些模块依赖的lib库
    yum -y install pcre pcre-devel
    yum -y install zlib zlib-devel
    yum -y install openssl openssl-devel
    #解压nginx
    tar -zxvf nginx-1.12.0.tar.gz
    #解压fastdfs-nginx-module-master
    yum -y install unzip
    unzip fastdfs-nginx-module-master.zip
    #安装nginx
    cd nginx-1.12.0
    #配置参考(Nginx安装.md)
    ./configure --prefix=/usr/local/Nginx --add-module=/usr/local/src/fastdfs-nginx-module-master/src
    make
    make install
    

    ./configure配置成功会看到如下信息:

    将http.conf和mime.types拷贝到/etc/fdfs目录下
    cp /usr/local/src/fastdfs-master/conf/http.conf /etc/fdfs/
    cp /usr/local/src/fastdfs-master/conf/mime.types /etc/fdfs/
    
    复制 fastdfs-nginx-module 源码中的配置文件到/etc/fdfs 目录,并修改
    cp /usr/local/src/fastdfs-nginx-module-master/src/mod_fastdfs.conf /etc/fdfs/
    #修改mod_fastdfs.conf
    vi /etc/fdfs/mod_fastdfs.conf
    #与当前Storage服务器storage.conf中的group_name一致
    group_name=group1
    #保存日志目录
    base_path=/usr/local/FastDFS/Storage
    #tracker服务器的IP地址以及端口号
    tracker_server=10.1.6.210:22122
    tracker_server=10.1.6.211:22122
    #文件 url 中是否有 group 名
    url_have_group_name = true
    #文件实际存储路径
    store_path0=/usr/local/FastDFS/FileData
    group_count = 2
    [group1]
    group_name=group1
    storage_server_port=23000
    store_path_count=1
    store_path0=/usr/local/FastDFS/FileData
    
    [group2]
    group_name=group2
    storage_server_port=23000
    store_path_count=1
    store_path0=/usr/local/FastDFS/FileData
    
    建立M00至存储目录的符号连接
    ln -s /usr/local/FastDFS/FileData/data /usr/local/FastDFS/FileData/data/M00
    
    配置nginx.conf
    vi /usr/local/Nginx/conf/nginx.conf
    #添加server
        server {
            listen       8888; #与/etc/fdfs/storage.conf 中的 http.server_port=8888 相对应
    
            # FastDFS 文件访问配置(fastdfs-nginx-module模块)
            location ~/group([0-9])/M00 {
                ngx_fastdfs_module;
            }
        }
    

    每台Storage都需要安装,配置的listen端口保持一致(不一致会出现一个组内的nginx访问另外一个组的无法访问。当天上传跨组访问正常,周末过后周一在尝试无法跨组访问不知道为什么,难道是mod_fastdfs.conf不路由非当天上传文件。)。
    group1的Storage的Nginx访问
    http://10.1.6.212:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
    http://10.1.6.212:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg
    http://10.1.6.213:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
    http://10.1.6.213:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg

    group2的Storage的Nginx访问
    http://10.1.6.214:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
    http://10.1.6.214:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg
    http://10.1.6.215:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
    http://10.1.6.215:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg

    10.1.6.212与10.1.6.213是group1

    10.1.6.214与10.1.6.215是group2

    5、Tracker在跟踪器节点(10.1.6.210、10.1.6.211)上 安装 Nginx


    #安装nginx需要的一些模块依赖的lib库
    yum -y install pcre pcre-devel
    yum -y install zlib zlib-devel
    yum -y install openssl openssl-devel
    #解压ngx_cache_purge
    cd /usr/local/src
    tar -zxvf ngx_cache_purge-2.3.tar.gz
    #解压nginx
    tar -zxvf nginx-1.12.0.tar.gz
    #编译安装nginx(添加ngx_cache_purge模块)
    cd nginx-1.12.0
    #配置参考(Nginx安装.md)
    ./configure --prefix=/usr/local/nginx --add-module=/usr/local/src/ngx_cache_purge-2.3
    make && make install
    
    配置nginx.conf
    vi /etc/nginx/nginx.conf
    #配置如下
    user  root; #root防止404错误
    worker_processes  1;
    
    #error_log  logs/error.log;
    #error_log  logs/error.log  notice;
    #error_log  logs/error.log  info;
    
    #pid        logs/nginx.pid;
    
    
    events {
        worker_connections  1024;
    }
    
    
    http {
        include       mime.types;
        default_type  application/octet-stream;
    
        #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
        #                  '$status $body_bytes_sent "$http_referer" '
        #                  '"$http_user_agent" "$http_x_forwarded_for"';
    
        #access_log  logs/access.log  main;
    
        sendfile        on;
        #tcp_nopush     on;
    
        #keepalive_timeout  0;
        keepalive_timeout  65;
    
        #gzip  on;
    
        #设置缓存
        server_names_hash_bucket_size 128;
        client_header_buffer_size 32k;
        large_client_header_buffers 4 32k;
        client_max_body_size 300m;
        proxy_redirect off;
        proxy_set_header Host $http_host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_connect_timeout 90;
        proxy_send_timeout 90;
        proxy_read_timeout 90;
        proxy_buffer_size 16k;
        proxy_buffers 4 64k;
        proxy_busy_buffers_size 128k;
        proxy_temp_file_write_size 128k; #设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限 
        proxy_temp_path /var/cache/nginx/proxy_temp/tmp;
        proxy_cache_path /var/cache/nginx/proxy_temp levels=1:2 keys_zone=http-cache:200m inactive=1d max_size=30g;
    
        #设置 group1 的服务器
        upstream fdfs_group1 {
             server 10.1.6.212:9999 weight=1 max_fails=2 fail_timeout=30s;
             server 10.1.6.213:9999 weight=1 max_fails=2 fail_timeout=30s;
        }
    
        #设置 group2 的服务器
        upstream fdfs_group2 {
             server 10.1.6.214:9999 weight=1 max_fails=2 fail_timeout=30s;
             server 10.1.6.215:9999 weight=1 max_fails=2 fail_timeout=30s;
        }
    
        server {
            listen       80;
            server_name  localhost;
    
            #charset koi8-r;
    
            #access_log  logs/host.access.log  main;
    
            location / {
                root   html;
                index  index.html index.htm;
            }
    
            #error_page  404              /404.html;
    
            # redirect server error pages to the static page /50x.html
            #
            error_page   500 502 503 504  /50x.html;
            location = /50x.html {
                root   html;
            }
    
            #设置 group 的负载均衡参数
            location /group1/M00 {
                proxy_next_upstream http_502 http_504 error timeout invalid_header;
                proxy_cache http-cache;
                proxy_cache_valid  200 304 12h;
                proxy_cache_key $uri$is_args$args;
                proxy_pass http://fdfs_group1;
                expires 30d;
            }
    
            location /group2/M00 {
                proxy_next_upstream http_502 http_504 error timeout invalid_header; proxy_cache http-cache;
                proxy_cache_valid 200 304 12h;
                proxy_cache_key $uri$is_args$args;
                proxy_pass http://fdfs_group2;
                expires 30d;
            }
    
            #设置清除缓存的访问权限
            location ~/purge(/.*) {
                #allow 127.0.0.1;
                allow all;
                deny all;
                proxy_cache_purge http-cache $1$is_args$args;
            }
    
            # proxy the PHP scripts to Apache listening on 127.0.0.1:80
            #
            #location ~ .php$ {
            #    proxy_pass   http://127.0.0.1;
            #}
    
            # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
            #
            #location ~ .php$ {
            #    root           html;
            #    fastcgi_pass   127.0.0.1:9000;
            #    fastcgi_index  index.php;
            #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
            #    include        fastcgi_params;
            #}
    
            # deny access to .htaccess files, if Apache's document root
            # concurs with nginx's one
            #
            #location ~ /.ht {
            #    deny  all;
            #}
        }
    
    
        # another virtual host using mix of IP-, name-, and port-based configuration
        #
        #server {
        #    listen       8000;
        #    listen       somename:8080;
        #    server_name  somename  alias  another.alias;
    
        #    location / {
        #        root   html;
        #        index  index.html index.htm;
        #    }
        #}
    
    
        # HTTPS server
        #
        #server {
        #    listen       443 ssl;
        #    server_name  localhost;
    
        #    ssl_certificate      cert.pem;
        #    ssl_certificate_key  cert.key;
    
        #    ssl_session_cache    shared:SSL:1m;
        #    ssl_session_timeout  5m;
    
        #    ssl_ciphers  HIGH:!aNULL:!MD5;
        #    ssl_prefer_server_ciphers  on;
    
        #    location / {
        #        root   html;
        #        index  index.html index.htm;
        #    }
        #}
    
    }
    
    #按配置要求,创建对应缓存目录
    mkdir -p /var/cache/nginx/proxy_temp
    mkdir -p /var/cache/nginx/proxy_temp/tmp
    

    6、配置Keepalived + Nginx高可用


    10.1.6.216、10.1.6.217

    keepalived安装:http://www.cnblogs.com/ddrsql/p/7075414.html
    nginx安装:http://www.cnblogs.com/ddrsql/p/7077622.html、

    nginx.conf配置
    vi /etc/nginx/nginx.conf
    #配置如下
    user  root;
    worker_processes  1;
    events {
        worker_connections  1024;
    }
    http {
        include       mime.types;
        default_type  application/octet-stream;
        sendfile        on;
        keepalive_timeout  65;
    
        ## FastDFS Tracker Proxy
        upstream fastdfs_tracker {
             server 10.1.6.210:80 weight=1 max_fails=2 fail_timeout=30s;
             server 10.1.6.211:80 weight=1 max_fails=2 fail_timeout=30s;
        }
    
        server {
            listen       80;
            server_name  localhost;
            location / {
                root html;
                index  index.html index.htm;
            }
            error_page   500 502 503 504  /50x.html;
            location = /50x.html {
                root html;
            }
    
            ## FastDFS Proxy
            location /dfs {
                root   html;
                index  index.html index.htm;
                proxy_pass  http://fastdfs_tracker/;
                proxy_set_header Host  $http_host;
                proxy_set_header Cookie $http_cookie;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                client_max_body_size  300m;
            }
        }
    }
    
    keepalived.conf配置

    VI_1与VI_2的"state"值在另外一台机器互换。
    10.1.6.216(VI_1:state MASTER; VI_2:state BACKUP)
    10.1.6.217(VI_1:state BACKUP; VI_2:state MASTER)

    vi /etc/keepalived/keepalived.conf
    #配置如下
    ! Configuration File for keepalived
    
    vrrp_script nginx {
        script "/etc/keepalived/nginx.sh"
        interval 2 # 检测间隔2s
        weight -5 # 若检测失败权重减低5
        fall 3 # 检测失败3次就定义为down状态
        rise 2 # 检测失败后,检测成功超过2次就定义为up状态
    }
    
    vrrp_instance VI_1 {
        state MASTER # master_server
        interface eth0
        virtual_router_id 51
        priority 100 # 权重值,值大的优先级高
        advert_int 2 # 检测时间间隔2s
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        virtual_ipaddress {
            10.1.6.218 # VIP
        }
        track_script {
           nginx # 检测脚本
        }
    }
    
    vrrp_instance VI_2 {
        state BACKUP # backup_server
        interface eth0
        virtual_router_id 52
        priority 98
        advert_int 2
        authentication {
            auth_type PASS
            auth_pass 1111
        }
        virtual_ipaddress {
            10.1.6.219
        }
        track_script {
           nginx
        }
    }
    
    
    nginx.sh
    vi /etc/keepalived/nginx.sh
    #添加执行权限
    chmod +x /etc/keepalived/nginx.sh
    #配置如下
    #!/bin/bash
    status=$(ss -lnp | grep -c 'nginx')
    if [ ${status} == 0 ]; then
        service nginx start
        sleep 2
        status=$(ss -lnp | grep -c 'nginx')
        if [ ${status} == 0 ]; then
            service keepalived stop
        fi
    fi
    

    通过虚拟ip访问
    http://10.1.6.218/dfs/group2/M00/00/00/CgEG11la-OqEBSdfAAAAAJk6Ln4052.jpg

  • 相关阅读:
    C# 图片与Base64的相互转化
    LeetCode 303. Range Sum Query – Immutable
    LeetCode 300. Longest Increasing Subsequence
    LeetCode 292. Nim Game
    LeetCode 283. Move Zeroes
    LeetCode 279. Perfect Squares
    LeetCode 268. Missing Number
    LeetCode 264. Ugly Number II
    LeetCode 258. Add Digits
    LeetCode 257. Binary Tree Paths
  • 原文地址:https://www.cnblogs.com/ddrsql/p/7118695.html
Copyright © 2011-2022 走看看