分布式集群搭建结构
- 双Tracker
- 2组Group
- 轮询存储策略
- Keepalived+Nginx高可用
- Nginx缓存
- 4个存储节点
一、 集群规划清单
1.安装清单
| 软件名称 | 版本 | 百度云盘存放名称 |
|---|---|---|
| FastDFS | 5.11 | fastdfs-5.11.zip |
| FastDFS-Nginx-module | 无 | fastdfs-nginx-module-master.zip |
| LibFastCommon | 1.0.36 | libfastcommon-1.0.36.zip |
| nginx | 1.10.3 | nginx-1.10.3.tar.gz |
| nginx-pure-cache | 2.3 | ngx_cache_purge-2.3.tar.gz |
安装所需文件均上传到百度云盘，位置：FastDFS百度云盘
2.集群规划
| 虚拟机 | IP | 说明 |
|---|---|---|
| Keepalived+Nginx1[Master] | 192.168.43.101 | Nginx Server 01 |
| Keeepalived+Nginx[Backup] | 192.168.43.102 | Nginx Server 02 |
| VIP | 192.168.43.150 | 虚拟漂移IP |
| Tracker01 | 192.168.43.70 | Tracker01服务器 |
| Tracker02 | 192.168.43.71 | Tracker02服务器 |
| Storage01 | 192.168.43.72 | Storage01服务器【group1】 |
| Storage02 | 192.168.43.73 | Storage02服务器【group1】 |
| Storage03 | 192.168.43.74 | Storage03服务器【group2】 |
| Storage04 | 192.168.43.75 | Storage04服务器【group2】 |
整体架构图如下图所示：
图片来源：CSDN作者 liuyazhuang
二、集群安装
以下操作均在关闭所有节点防火墙进行的，请根据个人情况开启相关端口或关闭防火墙
1.安装LibFastCommon/FastDFS模块
执行节点 Tracker01、Tracker02、Storage01、Storage03、Storage04
[root@localhost fastDFS]# unzip libfastcommon-1.0.36.zip
解压后目录如下：
[root@localhost fastdfs-5.11]# ll
[root@localhost libfastcommon-1.0.36]# ll
总用量 32
drwxr-xr-x. 2 root root 117 4月 5 2017 doc
-rw-r--r--. 1 root root 8005 4月 5 2017 HISTORY
-rw-r--r--. 1 root root 566 4月 5 2017 INSTALL
-rw-r--r--. 1 root root 1606 4月 5 2017 libfastcommon.spec
-rwxr-xr-x. 1 root root 3099 4月 5 2017 make.sh
drwxr-xr-x. 2 root root 191 4月 5 2017 php-fastcommon
-rw-r--r--. 1 root root 2763 4月 5 2017 README
drwxr-xr-x. 3 root root 4096 1月 17 11:21 src
安装C编译工具 gcc
[root@localhost fastdfs-5.11]# yum -y install gcc-c++
- 安装装LibFastCommon
编译libfastcommon软件并安装
[root@localhost fastdfs-5.11]# ./make.sh && ./make.sh install
为libcommon 创建软链接到/usr/local/lib目录下
[root@localhost fastdfs-5.11]# ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
[root@localhost fastdfs-5.11]# ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so
[root@localhost fastdfs-5.11]# ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so
[root@localhost fastdfs-5.11]# ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so
- 安装FastDFS
解压安装包
[root@localhost fastDFS]# unzip fastdfs-5.11.zip
进入解压目录并进行编译和安装
[root@localhost fastDFS]# cd fastdfs-5.11/
[root@localhost fastdfs-5.11]# ./make.sh && ./make.sh install
安装成功后，FastDFS会安装在/etc/fdfs目录下：
[root@localhost fastdfs-5.11]# ll /etc/fdfs/
总用量 76
-rw-r--r--. 1 root root 316 1月 17 11:47 client.conf
-rw-r--r--. 1 root root 1461 1月 17 11:25 client.conf.sample
-rw-r--r--. 1 root root 955 1月 17 13:20 http.conf
-rw-r--r--. 1 root root 31172 1月 17 13:21 mime.types
-rw-r--r--. 1 root root 3716 1月 17 12:57 mod_fastdfs.conf
-rw-r--r--. 1 root root 1278 1月 17 11:40 storage.conf
-rw-r--r--. 1 root root 7927 1月 17 11:25 storage.conf.sample
-rw-r--r--. 1 root root 105 1月 17 11:25 storage_ids.conf.sample
-rw-r--r--. 1 root root 1356 1月 17 11:34 tracker.conf
-rw-r--r--. 1 root root 7389 1月 17 11:25 tracker.conf.sample
我们需要把这三个示例文件复制一份，去掉.sample
[root@localhost fdfs]# cp client.conf.sample client.conf
[root@localhost fdfs]# cp storage.conf.sample storage.conf
[root@localhost fdfs]# cp tracker.conf.sample tracker.conf
FastDFS安装结束
2.安装Tracker并实现节点信息配置
执行节点 Tracker01、Tracker02
- 创建tracker工作目录
此目录用于保存tracker 的data和log
[root@localhost fdfs]# mkdir /opt/fastdfs_tracker
- 配置tracker
配置 /etc/fdfs目录下tracker.conf
主要实现以下5个配置内容：
1.disabled=false
2.port=22122 #默认端口号
3.base_path=/opt/fastdfs_tracker #我刚刚创建的目录
4.http.server_port=8080 #默认端口是8080
5.store_lookup=0 #采用轮询策略进行存储，0 轮询 1：始终定向到某个group 2：负载进行存储文件
完整tracker.conf 文件信息如下：
disabled=false
bind_addr= 0.0.0.0
port=22122
connect_timeout=30
network_timeout=60
base_path=/opt/fastdfs_tracker
max_connections=512
accept_threads=1
work_threads=4
min_buff_size = 8KB
max_buff_size = 128KB
store_lookup=0
store_group=group2
store_server=0
store_path=0
download_server=0
reserved_storage_space = 10%
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
sync_log_buff_interval = 10
check_active_interval = 120
thread_stack_size = 64KB
storage_ip_changed_auto_adjust = true
storage_sync_file_max_delay = 86400
storage_sync_file_max_time = 300
use_trunk_file = false
slot_min_size = 256
slot_max_size = 16MB
trunk_file_size = 64MB
trunk_create_file_advance = false
trunk_create_file_time_base = 02:00
trunk_create_file_interval = 86400
trunk_create_file_space_threshold = 20G
trunk_init_check_occupying = false
trunk_init_reload_from_binlog = false
trunk_compress_binlog_min_interval = 0
use_storage_id = false
storage_ids_filename = storage_ids.conf
id_type_in_filename = ip
store_slave_file_use_link = false
rotate_error_log = false
error_log_rotate_time=00:00
rotate_error_log_size = 0
log_file_keep_days = 0
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.server_port=8080
http.check_alive_interval=30
http.check_alive_type=tcp
http.check_alive_uri=/status.html
修改保存后创建软引用
[root@localhost fdfs]# ln -s /usr/bin/fdfs_storaged /usr/local/bin
- 启动tracker，并加入开机启动项
[root@localhost fdfs]# service fdfs_trackerd start
将tracker加入开机启动项
[root@localhost fdfs]# echo "service fdfs_trackerd start" |tee -a /etc/rc.d/rc.local
3.安装Storage模块并实现配置
执行节点 Storage01、Storage02、Storage03、Storage04
- 建立存储目录
在存储各节点建了两个目录fastdfs_storage_data,fastdfs_storage
[root@localhost opt]# mkdir fastdfs_storage
[root@localhost opt]# mkdir fastdfs_storage_data
[root@localhost opt]# ll
总用量 0
drwxr-xr-x. 4 root root 30 1月 17 11:45 fastdfs_storage
drwxr-xr-x. 3 root root 18 1月 17 11:45 fastdfs_storage_data
drwxr-xr-x. 4 root root 30 1月 17 11:35 fastdfs_tracker
- 修改存储节点目录下/etc/fdfs/storage.conf配置信息，具体如下：
disabled=false #启用配置文件
group_name=group1 #组名（第一组为 group1， 第二组为 group2）
port=23000 #storage 的端口号,同一个组的 storage 端口号必须相同
base_path=/opt/fastdfs_storage #设置storage数据文件和日志目录
store_path0=/opt/fastdfs_storage_data #实际文件存储路径
store_path_count=1 #存储路径个数，需要和 store_path 个数匹配
tracker_server=192.168.43.70:22122 #tracker 服务器的 IP 地址和端口
tracker_server=192.168.43.70:22122 #多个 tracker 直接添加多条配置
http.server_port=8888 #设置 http 端口号
完整配置信息如下：
disabled=false
group_name=group1
bind_addr=
client_bind=true
port=23000
connect_timeout=30
network_timeout=60
heart_beat_interval=30
stat_report_interval=60
base_path=/opt/fastdfs_storage
max_connections=256
buff_size = 256KB
accept_threads=1
work_threads=4
disk_rw_separated = true
disk_reader_threads = 1
disk_writer_threads = 1
sync_wait_msec=50
sync_interval=0
sync_start_time=00:00
sync_end_time=23:59
write_mark_file_freq=500
store_path_count=1
store_path0=/opt/fastdfs_storage_data
subdir_count_per_path=256
tracker_server=192.168.43.70:22122
tracker_server=192.168.43.71:22122
log_level=info
run_by_group=
run_by_user=
allow_hosts=*
file_distribute_path_mode=0
file_distribute_rotate_count=100
fsync_after_written_bytes=0
sync_log_buff_interval=10
sync_binlog_buff_interval=10
sync_stat_file_interval=300
thread_stack_size=512KB
upload_priority=10
if_alias_prefix=
check_file_duplicate=0
file_signature_method=hash
key_namespace=FastDFS
keep_alive=0
use_access_log = false
rotate_access_log = false
access_log_rotate_time=00:00
rotate_error_log = false
error_log_rotate_time=00:00
rotate_access_log_size = 0
rotate_error_log_size = 0
log_file_keep_days = 0
file_sync_skip_invalid_record=false
use_connection_pool = false
connection_pool_max_idle_time = 3600
http.domain_name=
http.server_port=8888
- 启动Storage
各节点配置好信息好之后，启动Storage
[root@localhost fdfs]# service fdfs_storaged start
启动后查看日志情况
[root@localhost fdfs]# fdfs_monitor /etc/fdfs/storage.conf
[2018-01-20 16:56:48] DEBUG - base_path=/opt/fastdfs_storage, connect_timeout=30, network_timeout=60, tracker_server_count=2, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0
server_count=2, server_index=0
tracker server is 192.168.43.70:22122
group count: 2
Group 1:
group name = group1
disk total space = 47073 MB
disk free space = 35162 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0
Storage 1:
id = 192.168.43.72
ip_addr = 192.168.43.72 (localhost.localdomain) ACTIVE
http domain =
version = 5.11
join time = 2018-01-19 13:59:30
up time = 2018-01-20 12:37:18
total storage = 47073 MB
free storage = 35162 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8888
current_write_path = 0
source storage id =
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 3
success_upload_count = 3
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 791904
success_upload_bytes = 791904
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 775234
success_sync_in_bytes = 775234
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 4
success_file_open_count = 4
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 8
success_file_write_count = 8
last_heart_beat_time = 2018-01-20 16:56:18
last_source_update = 2018-01-19 19:34:55
last_sync_update = 2018-01-19 15:28:56
last_synced_timestamp = 2018-01-19 15:28:48 (0s delay)
Storage 2:
id = 192.168.43.73
ip_addr = 192.168.43.73 ACTIVE
http domain =
version = 5.11
join time = 2018-01-19 14:00:21
up time = 2018-01-20 12:37:42
total storage = 47073 MB
free storage = 35166 MB
upload priority = 10
store_path_count = 1
subdir_count_per_path = 256
storage_port = 23000
storage_http_port = 8888
current_write_path = 0
source storage id = 192.168.43.72
if_trunk_server = 0
connection.alloc_count = 256
connection.current_count = 1
connection.max_count = 1
total_upload_count = 1
success_upload_count = 1
total_append_count = 0
success_append_count = 0
total_modify_count = 0
success_modify_count = 0
total_truncate_count = 0
success_truncate_count = 0
total_set_meta_count = 0
success_set_meta_count = 0
total_delete_count = 0
success_delete_count = 0
total_download_count = 0
success_download_count = 0
total_get_meta_count = 0
success_get_meta_count = 0
total_create_link_count = 0
success_create_link_count = 0
total_delete_link_count = 0
success_delete_link_count = 0
total_upload_bytes = 775234
success_upload_bytes = 775234
total_append_bytes = 0
success_append_bytes = 0
total_modify_bytes = 0
success_modify_bytes = 0
stotal_download_bytes = 0
success_download_bytes = 0
total_sync_in_bytes = 791904
success_sync_in_bytes = 791904
total_sync_out_bytes = 0
success_sync_out_bytes = 0
total_file_open_count = 4
success_file_open_count = 4
total_file_read_count = 0
success_file_read_count = 0
total_file_write_count = 8
success_file_write_count = 8
last_heart_beat_time = 2018-01-20 16:56:42
last_source_update = 2018-01-19 15:28:48
last_sync_update = 2018-01-19 19:34:59
last_synced_timestamp = 2018-01-19 19:34:55 (0s delay)
Group 2:
group name = group2
disk total space = 47073 MB
disk free space = 35165 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port =