Linux系统运维之FastDFS集群部署

一、简介

　　FastDFS是一个开源的轻量级分布式文件系统，它对文件进行管理，功能包括：文件存储、文件同步、文件访问（文件上传、文件下载）等，解决了大容量存储和负载均衡的问题。FastDFS服务端有两个角色：跟踪器（tracker）和存储节点（storage）。跟踪器主要做调度工作，在访问上起负载均衡的作用。

　　存储节点存储文件，完成文件管理的所有功能：就是这样的存储、同步和提供存取接口，FastDFS同时对文件的metadata进行管理。所谓文件的meta data就是文件的相关属性，以键值对（key valuepair）方式表示，如：width=1024，其中的key为width，value为1024。文件metadata是文件属性列表，可以包含多个键值对。

二、拓扑

　　　　　

三、集群部署

1.安装开发环境

　　将下列命令在所有服务器上执行：

[root@LFTp-FastDFS01 opt]# yum install -y zlib zlib-devel pcre pcre-devel gcc gcc-c++ openssl openssl-devel libevent libevent-devel perl unzip

　　或者：

[root@LFTp-FastDFS01 ~]# yum groupinstall 'Development tools' -y

2.创建数据储存目录

　　在tracker_server上创建tracker目录；在storage_server上创建storage目录。

[root@LFTp-FastDFS01 ~]# mkdir -p /data/fastdfs/{storage,tracker}

3.下载软件包

　　下载Libfastcommon:

[root@LFTp-FastDFS01 ~]# wget https://github.com/happyfish100/libfastcommon/archive/master.zip

　　下载FastDFS:

[root@LFTp-FastDFS01 ~]# wget https://github.com/happyfish100/fastdfs/archive/V5.05.tar.gz

　　下载FastDFS-Nginx模块：

[root@LFTp-FastDFS01 ~]# wget http://jaist.dl.sourceforge.net/project/fastdfs/FastDFS%20Nginx%20Module%20Source%20Code/fastdfs-nginx-module_v1.16.tar.gz

　　下载Nginx：

[root@LFTp-FastDFS01 ~]# wget http://nginx.org/download/nginx-1.8.0.tar.gz

　　ehomepay内网下载：

[root@LFTp-FastDFS01 ~]# mkdir -p /soft/FastDFS
[root@LFTp-FastDFS01 ~]# cd /soft/FastDFS
[root@LFTp-FastDFS01 FastDFS]# wget -r -np -nd http://10.12.3.24/soft/FastDFS/

4. 安装Libfastcommon

　　Tracker-server和storage-server都需要安装：

[root@LFTp-FastDFS01 libfastcommon-master]# unzip libfastcommon-master.zip
[root@LFTp-FastDFS01 libfastcommon-master]# cd  libfastcommon-master
[root@LFTp-FastDFS01 libfastcommon-master]# ./make.sh
[root@LFTp-FastDFS01 libfastcommon-master]# ./make.sh install
[root@LFTp-FastDFS01 etc]# ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
[root@LFTp-FastDFS01 etc]# ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so
[root@LFTp-FastDFS01 etc]# ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so
[root@LFTp-FastdFS01 etc]# ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so

5.安装FastDFS

　　解压、编译安装FastDFS

[root@LFTp-FastDFS01 FastDFS]# tar -xf  FastDFS_v5.08.tar.gz
[root@LFTp-FastDFS01 FastDFS]# cd FastDFS
[root@LFTp-FastDFS01 FastDFS]# sh make.sh && sh make.sh install
[root@LFTp-FastDFS01 FastDFS]# cp -pa conf/*.conf /etc/fdfs/
[root@LFTp-FastDFS02 FastDFS]# cp -pa conf/mime.types /etc/fdfs/
[root@LFTp-FastDFS01 FastDFS]# cd  /etc/fdfs/
[root@LFTp-FastDFS01 fdfs]# rm -rf *.sample
[root@LFTp-FastDFS01 fdfs]# chown -R nobody.nobody /data/fastdfs
[root@LFTp-FastDFS01 fdfs]# chown -R nobody.nobody /etc/fdfs

注：这里赋予nobody账号权限是因为nginx用nobody用户启动

　　修改tracker.conf配置：

disabled=false
bind_addr=10.12.8.42
port=22122
connect_timeout=30
network_timeout=60
base_path=/data/fastdfs/tracker
max_connections=256
accept_threads=1
work_threads=4
store_lookup=2
store_group=group1
store_server=0
store_path=0
download_server=0
reserved_storage_space = 10%
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
sync_log_buff_interval = 10
check_active_interval = 120
thread_stack_size = 64KB
storage_ip_changed_auto_adjust = true
storage_sync_file_max_delay = 86400
storage_sync_file_max_time = 300
use_trunk_file = false 
slot_min_size = 256
slot_max_size = 16MB
trunk_file_size = 64MB
trunk_create_file_advance = false
trunk_create_file_time_base = 02:00
trunk_create_file_interval = 86400
trunk_create_file_space_threshold = 20G
trunk_init_check_occupying = false
trunk_init_reload_from_binlog = false
trunk_compress_binlog_min_interval = 0
use_storage_id = false
storage_ids_filename = storage_ids.conf
id_type_in_filename = ip
store_slave_file_use_link = false
rotate_error_log = false
error_log_rotate_time=00:00
rotate_error_log_size = 0
log_file_keep_days = 0
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.server_port=80
http.check_alive_interval=30
http.check_alive_type=tcp
http.check_alive_uri=/status.html

　　修改storage.conf配置：

disabled=false
group_name=group1(这里storage属于group1,如有多个group根据实际情况填写)
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/data/fastdfs/storage
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/data/fastdfs/storage
subdir_count_per_path=256
tracker_server=10.12.8.42:22122(tracker地址,可写多个)
tracker_server=10.12.8.44:22122(tracker地址,可写多个)
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=80

　　修改mod_fastdfs.conf配置：

connect_timeout=2
network_timeout=30
base_path=/data/fastdfs/storage
load_fdfs_parameters_from_tracker=true
storage_sync_file_max_delay = 86400
use_storage_id = false
storage_ids_filename = storage_ids.conf
tracker_server=10.12.8.42:22122
storage_server_port=23000
group_name=group1
url_have_group_name = true
store_path_count=1
store_path0=/data/fastdfs/storage
log_level=info
log_filename=
response_mode=proxy
if_alias_prefix=
flv_support = true
flv_extension = flv
group_count = 0

6.安装Nginx和Fastdfs-Nginx模块

　　需要通过nginx进行访问，安装nginx-fastdfs模块和nginx：

[root@LFTp-FastDFS01 FastDFS]# tar -zxf fastdfs-nginx-module_v1.16.tar.gz && tar -zxf nginx-1.10.1.tar.gz
[root@LFTp-FastDFS01 FastDFS]# cd fastdfs-nginx-module/src/
[root@LFTp-FastDFS01 src]# cp -r mod_fastdfs.conf /etc/fdfs/
#修改config文件如下
更改前：“CORE_INCS="$CORE_INCS /usr/local/include/fastdfs /usr/local/include/fastcommon/"
CORE_LIBS="$CORE_LIBS -L/usr/local/lib -lfastcommon -lfdfsclient"”
更改后：“CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/local/include/fastcommon/"
CORE_LIBS="$CORE_LIBS -L/usr/lib64 -lfastcommon -lfdfsclient"”

　　还可以做以下操作解决：

ln -sv /usr/incloud/fastcommon /usr/local/incloud/fastcommon
ln -sv /usr/incloud/fastdfs /usr/local/incloud/fastdfs
ln -sv /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so　

　　安装nginx:

[root@LFTp-FastDFS01 FastDFS]# cd nginx-1.10.1
[root@LFTp-FastDFS01 nginx-1.10.1]# ./configure --add-module=/soft/FastDFS/fastdfs-nginx-module/src/ --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module --with-http_realip_module
[root@LFTp-FastDFS01 nginx-1.10.1]# make && make install

　　修改nginx.conf配置：

user  nobody;
worker_processes  8;
worker_rlimit_nofile 65535;
#error_log  logs/error.log;
pid        logs/nginx.pid;
events {
    worker_connections  65535;
        use epoll;
        }
http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for" "$request_time"';
    #access_log  logs/access.log  main;
    sendfile        on;
    tcp_nopush     on;
    keepalive_timeout  65;
    client_max_body_size 64M;
    gzip  on;                                            #开启gzip
    gzip_min_length 1024;
    gzip_buffers 4 8k;
    gzip_types application/x-javascript text/css application/xml;
    fastcgi_connect_timeout 300;
    fastcgi_read_timeout 300;
    fastcgi_send_timeout 300;
    #fastcgi_buffer_size 64k;
    #fastcgi_buffers   4 32k;
    #fastcgi_busy_buffers_size 64k;
    #fastcgi_temp_file_write_size 64k;
    ##proxy_ignore_client_abort on;
    #
    #安全需要禁止缓存
    #expires    6h;
    #add_header Pragma no-cache;
    #add_header Cache-Control no-cache;
    #add_header Cache-Control no-store;
    #add_header Cache-Control must-revalidate;
    #安全需要禁止iFrame,防止iFrame嵌入
    #add_header X-Frame-Options SAMEORIGIN;    
    include vhost/*.conf;
}

　　增加vhost虚拟主机：

server {
        listen 80 default;
        server_name 10.12.8.42;
        access_log  logs/access.v5ehomepay.log  main;
        error_log  logs/error.v5ehomepay.log;
        location /nginx-status {
           stub_status on;
           access_log on;
        }
        location /group1/M00 {
            root /data/fastdfs/storage/data/;
            ngx_fastdfs_module;
          }
        location ~ .*.(gif|jpg|jpeg|png|bmp|swf|js)$ {
#      fastcgi_cache cache_one;  #nginx.conf 开启cache才行，要不然启动报错
       fastcgi_cache_valid 200 10m;
       fastcgi_cache_valid 304 3m;
       fastcgi_cache_valid 301 302 1h;
       fastcgi_cache_valid any 1m;
       fastcgi_cache_min_uses 1;
       fastcgi_cache_use_stale error timeout invalid_header http_500;
       fastcgi_cache_key $host$request_uri;
       access_log off;
       }
        error_page 403  404 /40x.html;
        location = /40x.html             root   html;
        }
        error_page   404 500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }

7.启动进程，并查看端口

[root@LFTp-FastDFS01 ~]# /etc/init.d/fdfs_trackerd start
[root@LFTp-FastDFS01 ~]# /etc/init.d/fdfs_storaged start
Starting FastDFS storage server: 
[root@LFTp-FastDFS01 ~]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
#查看进程
[root@LFTp-FastDFS01 ~]# netstat -npl|grep -E "nginx|fdfs"
tcp        0      0 0.0.0.0:23000               0.0.0.0:*                   LISTEN      27454/fdfs_storaged 
tcp        0      0 10.12.8.42:22122            0.0.0.0:*                   LISTEN      14179/fdfs_trackerd 
tcp        0      0 0.0.0.0:80                  0.0.0.0:*                   LISTEN      14201/nginx      

注意事情：Centos 7.1版本中trackerd和storaged启动失败时，做以下操作

ln -s /usr/bin/fdfs_trackerd /usr/local/bin
ln -s /usr/bin/fdfs_storaged /usr/local/bin
ln -s /usr/bin/stop.sh /usr/local/bin
ln -s /usr/bin/restart.sh /usr/local/bin

8.测试上传下载功能，确认完成！

[root@LFTp-FastDFS01 ~]# echo 'test fastDFS!!!' > test.htm
[root@LFTp-FastDFS01 ~]# vim /etc/fdfs/client.conf
添加tracker_server地址
[root@LFTp-FastDFS01 ~]# fdfs_test /etc/fdfs/client.conf upload test.html 
This is FastDFS client test program v5.08
Copyright (C) 2008, Happy Fish / YuQing
FastDFS may be copied only under the terms of the GNU General
Public License V3, which may be found in the FastDFS source kit.
Please visit the FastDFS Home Page http://www.csource.org/ 
for more detail.
[2016-08-03 15:40:38] DEBUG - base_path=/data/fastdfs/storage, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
tracker_query_storage_store_list_without_group: 
    server 1. group_name=, ip_addr=10.12.8.42, port=23000
    server 2. group_name=, ip_addr=10.12.8.43, port=23000
group_name=group1, ip_addr=10.12.8.43, port=23000
storage_upload_by_filename
group_name=group1, remote_filename=M00/00/00/CgwIK1ehn_aATP3dAAAAELIjPeQ51.html
source ip address: 10.12.8.43
file timestamp=2016-08-03 15:40:38
file size=16
file crc32=2988654052
example file url: http://10.12.8.43/group1/M00/00/00/CgwIK1ehn_aATP3dAAAAELIjPeQ51.html
storage_upload_slave_by_filename
group_name=group1, remote_filename=M00/00/00/CgwIK1ehn_aATP3dAAAAELIjPeQ51_big.html
source ip address: 10.12.8.43
file timestamp=2016-08-03 15:40:38
file size=16
file crc32=2988654052
example file url: http://10.12.8.43/group1/M00/00/00/CgwIK1ehn_aATP3dAAAAELIjPeQ51_big.html
[root@LFTp-FastDFS01 logs]# curl http://10.12.8.43/group1/M00/00/00/CgwIK1ehn_aATP3dAAAAELIjPeQ51_big.html
test fastDFS!!!