pyspider安装

环境准备

yum update -y
yum install gcc gcc-c++ -y   //安装开发编译工具
um install python-pip python-devel python-distribute libxml2 libxml2-devel python-lxml libxslt libxslt-devel openssl openssl-devel -y  //安装依赖库
pip install --upgrade pip  //升级pip 该步骤可选

部署mariadb

yum -y install mariadb-server mariadb
systemctl start mariadb  //启动mariadb服务
mysqladmin -u root password "Password"  //设置mariadb密码
检查mariadb是否安装成功

部署redis

wget http://download.redis.io/redis-stable.tar.gz
tar -zxvf redis-stable.tar.gz
mv redis-stable /usr/local/redis
cd /usr/local/redis/
make && make install
设置redis配置  设置配置文件路径
mkdir -p /etc/redis
cp /usr/local/redis/redis.conf /etc/redis/redis.conf
vim /etc/redis/redis.conf  修改
daemonize yes
启动 redis服务
/usr/local/bin/redis-server /etc/redis/redis.conf

部署pyspider

安装依赖
pip install --upgrade chardet
easy_install mysql-connector==2.1.3
easy_install redis
pip install pyspider    //安装

配置pyspider

mkdir /etc/pyspider
vim /etc/pyspider/pyspider.conf.json   //写入
{ 
  "taskdb": "mysql+taskdb://root:123.com@127.0.0.1:3306/taskdb",
  "projectdb": "mysql+projectdb://root:123.com@127.0.0.1:3306/projectdb",
  "resultdb": "mysql+resultdb://root:123.com@127.0.0.1:3306/resultdb",
  "message_queue": "redis://127.0.0.1:6379/db",
  "webui": {
    "username": "root",
    "password": "123.com",
    "need-auth": true
  }
}

启动服务

pyspider -c /etc/pyspider/pyspider.conf.json

http://ip:5000/   //查看