分布式文件系统的实现
1 配置数据库节点node5(172.16.21.5)
Mysql的cmake编译与安装，详细步步骤参照我的"Mysql的cmake编译与安装博文"
[root@node5 ~]# mysql
MariaDB [(none)]> create database mogilefs;
Query OK, 1 row affected (0.02 sec)
MariaDB [(none)]> use mogilefs
Database changed
#授权root用户所有权限
MariaDB [mogilefs]> grant all on *.* to 'root'@'172.16.21.5';
Query OK, 0 rows affected (0.00 sec)
#授权mogilefs的tracker连接数据库的用户
MariaDB [mogilefs]> grant all on *.* to 'moguser'@'172.16.%.%' identified by 'mogpass' with grant option;
Query OK, 0 rows affected (0.00 sec)
MariaDB [mogilefs]> flush privileges;
Query OK, 0 rows affected (0.00 sec)
2、安装配置mogilefs（三台主机: node2:172.16.21.2, node3:172.16.21.3 ,node4:172.16.21.4）
（1）安装
[root@node2 ~]# ls
anaconda-ks.cfg copyvbox.sh install.log.syslog mariadb-10.0.13.tar.gz mysql.sh
centos6.6.repo install.log mariadb-10.0.13 mogilefs percona-toolkit-2.2.4-1.noarch.rpm
[root@node2 ~]# cd mogilefs/
[root@node2 mogilefs]# ls
MogileFS-Server-2.46-2.el6.noarch.rpm Perlbal-doc-1.78-1.el6.noarch.rpm
MogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm perl-MogileFS-Client-1.14-1.el6.noarch.rpm
MogileFS-Server-mogstored-2.46-2.el6.noarch.rpm perl-Net-Netmask-1.9015-8.el6.noarch.rpm
MogileFS-Utils-2.19-1.el6.noarch.rpm perl-Perlbal-1.78-1.el6.noarch.rpm
Perlbal-1.78-1.el6.noarch.rpm
[root@node2 mogilefs]# yum -y install *.rpm perl-IO-AIO
（2）配置tracker
节点node2(172.16.21.2)
①、修改mogilefsd的配置文件
[root@node2 ~]#vim /etc/mogilefs/mogilefsd.conf
# Enable daemon mode to work in background and use syslog
daemonize = 1
# Where to store the pid of the daemon (must be the same in the init script)
pidfile = /var/run/mogilefsd/mogilefsd.pid
# Database connection information
db_dsn = DBI:mysql:mogilefs:host=172.16.21.5 #数据库
db_user = moguser #数据库授权的用户
db_pass = mogpass #用户密码
# IP:PORT to listen on for mogilefs client requests
listen = 172.16.21.2:7001 #监听的端口及IP
# Optional, if you don't define the port above.
conf_port = 7001
# Number of query workers to start by default.
query_jobs = 10
# Number of delete workers to start by default.
delete_jobs = 1
# Number of replicate workers to start by default.
replicate_jobs = 5
# Number of reaper workers to start by default.
# (you don't usually need to increase this)
reaper_jobs = 1
②、为mogilefsd设定数据库
[root@node2 ~]# mogdbsetup --dbhost=172.16.21.5 --dbname=mogilefs --dbrootuser=root --dbuser=moguser --dbpass=mogpass
This will attempt to setup or upgrade your MogileFS database.
It won't destroy existing data.
Run with --help for more information. Run with --yes to shut up these prompts.
Continue? [N/y]: y
[root@node2 ~]#
[root@node2 ~]# service mogilefsd start
Starting mogilefsd [ OK ]
③、查看7001端口是否成功监听
[root@node2 ~]# ss –tnlp
④、验证数据库(在节点node5:172.16.21.5)
[root@node5 ~]# mysql
MariaDB [(none)]> use mogilefs
Database changed
MariaDB [mogilefs]> show tables;
+----------------------+
| Tables_in_mogilefs |
+----------------------+
| checksum |
| class |
| device |
| domain |
| file |
| file_on |
| file_on_corrupt |
| file_to_delete |
| file_to_delete2 |
| file_to_delete_later |
| file_to_queue |
| file_to_replicate |
| fsck_log |
| host |
| server_settings |
| tempfile |
| unreachable_fids |
+----------------------+
17 rows in set (0.00 sec)
MariaDB [mogilefs]>
说明我在节点二上的设置的mogilefs已经开启
为了完整性，接下来继续在节点node2上操作：
创建一个磁盘分区：
fdisk /dev/sda
n p 3 +30G w
[root@node2 ~]# partx -a /dev/sda
磁盘初始化
[root@node2 ~]# mke2fs -t ext4 /dev/sda3
[root@node2 ~]# mkdir /mogilefs
接下来就是把磁盘/dev/sda3挂载 到 /mogilefs
[root@node2 ~]# vim /etc/fstab
/dev/sda3 /mogilefs ext4 defaults 0 0
[root@node2 ~]# mount –a
[root@node2 ~]# mount
[root@node2 ~]# chown -R mogilefs.mogilefs /mogilefs/
#########配置mogstored#########
[root@node2 ~]# vim /etc/mogilefs/mogstored.conf
maxconns = 10000
httplisten = 0.0.0.0:7500
mgmtlisten = 0.0.0.0:7501
docroot = /mogilefs
（4）查看7500端口监听状态
[root@node2 ~]# service mogstored restart
（5）添加主机
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host add 172.16.21.2 --ip=172.16.21.2 --status=alive
###查看添加主机命令
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host list
172.16.21.2 [1]: alive
IP: 172.16.21.2:7500
[root@node2 ~]#
（6）添加设备
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device add 172.16.21.2 1
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device list
172.16.21.2 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 0.042 27.866 27.908 100
[root@node2 ~]#
（7）添加domain
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain add images
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain add files
###列出domain###
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
（8）添加class
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class add images class1
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class add images class2
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images class1 2 MultipleHosts() NONE
images class2 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
（9）配置其它两台主机的mogilefsd和mogstored
因为这三台主机的tracker区用一个数据库，所以在安装配置时，不需再重新设定数据库，只配置相应的mogilefsd和mogstored配置文件就可以了
（10）在节点node2（172.16.21.2）查看最后的状态信息
①、列出所有主机
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host list
172.16.21.2 [1]: alive
IP: 172.16.21.2:7500
172.16.21.3 [2]: alive
IP: 172.16.21.3:7500
172.16.21.4 [3]: alive
IP: 172.16.21.4:7500
②、列出所有设备
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device list
172.16.21.2 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 0.043 27.865 27.908 100
172.16.21.3 [2]: alive
used(G) free(G) total(G) weight(%)
dev2: alive 0.043 27.865 27.908 100
172.16.21.4 [3]: alive
used(G) free(G) total(G) weight(%)
dev3: alive 0.043 27.865 27.908 100
③、列出所有domain
[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images class1 2 MultipleHosts() NONE
images class2 2 MultipleHosts() NONE
images class3 2 MultipleHosts() NONE
images class4 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
music default 2 MultipleHosts() NONE
note default 2 MultipleHosts() NONE
（11）上传图片资源到mogilefs
[root@node2 ~]# mogupload --trackers=172.16.21.2:7001 --domain=images --key='1.jpg' --file='/root/pictures/1.jpg'
（12）使用mogfileinfo获取上传图片的访问路径
[root@node2 ~]# moglistkeys --trackers=172.16.21.2:7001 --domain=images
1.jpg
[root@node2 ~]# mogfileinfo --trackers=172.16.21.2:7001 --domain=images --key='1.jpg'
- file: 1.jpg
class: default
devcount: 2
domain: images
fid: 5
key: 1.jpg
length: 104372
- http://172.16.21.2:7500/dev1/0/000/000/0000000005.fid
- http://172.16.21.3:7500/dev2/0/000/000/0000000005.fid
五、配置nginx服务器(172.16.21.1)
1、解压nginx_mogilefs_module-1.0.4.zip模块
[root@localhost ~]# unzip nginx_mogilefs_module-1.0.4.zip
lftp 172.16.0.1:/pub/Sources/sources/nginx> get nginx-1.6.2.tar.gz
[root@localhost ~]# tar xf nginx-1.6.2.tar.gz
一、安装Nginx：
1、解决依赖关系
# yum groupinstall "Development Tools" "Server Platform Deveopment"
# yum install openssl-devel pcre-devel
2、安装
首先添加用户nginx，实现以之运行nginx服务进程：
# groupadd -r nginx
# useradd -r -g nginx nginx
3开始编译：
[root@localhost ~]#cd nginx-1.6.2
接着开始编译和安装：
# ./configure
--prefix=/usr
--sbin-path=/usr/sbin/nginx
--conf-path=/etc/nginx/nginx.conf
--error-log-path=/var/log/nginx/error.log
--http-log-path=/var/log/nginx/access.log
--pid-path=/var/run/nginx/nginx.pid
--lock-path=/var/lock/nginx.lock
--user=nginx
--group=nginx
--with-http_ssl_module
--with-http_flv_module
--with-http_stub_status_module
--with-http_gzip_static_module
--http-client-body-temp-path=/var/tmp/nginx/client/
--http-proxy-temp-path=/var/tmp/nginx/proxy/
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi
--http-scgi-temp-path=/var/tmp/nginx/scgi
--with-pcre
--with-debug
--add-module=/root/nginx_mogilefs_module-1.0.4
# make && make install
4.为nginx提供SysV init脚本:
新建文件/etc/rc.d/init.d/nginx，内容如下：
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig: - 85 15
# description: Nginx is an HTTP(S) server, HTTP(S) reverse
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /etc/nginx/nginx.conf
# config: /etc/sysconfig/nginx
# pidfile: /var/run/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/usr/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() {
# make required directories
user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=([^ ]*).*/1/g' -`
options=`$nginx -V 2>&1 | grep 'configure arguments:'`
for opt in $options; do
if [ `echo $opt | grep '.*-temp-path'` ]; then
value=`echo $opt | cut -d "=" -f 2`
if [ ! -d "$value" ]; then
# echo "creating" $value
mkdir -p $value && chown -R $user $value
fi
fi
done
}
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
make_dirs
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
sleep 1
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit 0
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit 2
esac
5.而后为此脚本赋予执行权限：
# chmod +x /etc/rc.d/init.d/nginx
添加至服务管理列表，并让其开机自动启动：
# chkconfig --add nginx
# chkconfig nginx on
6.而后就可以启动服务并测试了：
# service nginx start
7、修改nginx配置文件
只需在location段中增加如下几行代码
location / {
mogilefs_tracker 172.16.21.3:7001;
mogilefs_domain images;
mogilefs_methods get;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
使用curl上传文件。
curl -X PUT -T '/etc/passwd' http://172.16.100.15/upload/passwd.html