zoukankan      html  css  js  c++  java
  • python3爬虫环境搭建

    安装python3

    sudo apt-get install python3-dev build-essential libssl-dev libffi-dev libxml2 libxml2-dev libxslt1-dev zlib1g-dev
    sudo apt-get install python3
    sudo apt-get install ptyhon3-pip
    

    ubuntu安装mongo

    sudo apt-get install mongodb
    mongod
    mongo
    > show dbs
    admin  (empty)
    local  0.078GB
    > use local
    switched to db local
    > db.test.insert({'a':'b'})
    WriteResult({ "nInserted" : 1 })
    

    mac安装mongo

    brew install mongodb
    brew services start mongodb
    

    ubuntu安装redis

    sudo apt-get install redis-server
    redis-cli
    127.0.0.1:6379> set 'a' 'b'
    OK
    127.0.0.1:6379> get 'a'
    "b"
    127.0.0.1:6379> 
    root@iZwz91u9ywulp6n78yyed6Z:~# sudo vi /etc/redis/redis.conf 
    requirepass myredisserver
    sudo service redis restart
    root@iZwz91u9ywulp6n78yyed6Z:~# redis-cli -a myredisserver
    127.0.0.1:6379> get 'a'
    "b"
    127.0.0.1:6379> 
    

    mac 安装redis

    brew install redis
    qunzhudeMacBook-Air:/ qunzhupu$ brew services start redis
    ==> Successfully started `redis` (label: homebrew.mxcl.redis)
    qunzhudeMacBook-Air:/ qunzhupu$ redis-cli
    127.0.0.1:6379> set 'a' 'b'
    OK
    127.0.0.1:6379> get 'a'
    "b"
    cd /usr/local/etc
    sudo vim redis.conf
    requirepass foobared
    

    ubuntu配置mysql

    root@iZwz91u9ywulp6n78yyed6Z:~# cd /etc/mysql/
    root@iZwz91u9ywulp6n78yyed6Z:/etc/mysql# ls
    conf.d  debian.cnf  debian-start  my.cnf  my.cnf.fallback  mysql.cnf  mysql.conf.d
    root@iZwz91u9ywulp6n78yyed6Z:/etc/mysql# cd mysql.conf.d/
    root@iZwz91u9ywulp6n78yyed6Z:/etc/mysql/mysql.conf.d# ls
    mysqld.cnf  mysqld_safe_syslog.cnf
    root@iZwz91u9ywulp6n78yyed6Z:/etc/mysql/mysql.conf.d# vi mysqld.cnf 
    root@iZwz91u9ywulp6n78yyed6Z:/etc/mysql/mysql.conf.d# sudo service mysql restart
    GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123456' WITH GRANT OPTION;
    FLUSH PRIVILEGES;
    

    selenium库使用

    >>> import selenium
    >>> from selenium import webdriver
    >>> driver - webdriver.Chrome()
    >>> driver.get('http://www.baidu.com')
    

    安装beautifulsoup4和pyquery

    pip3 install beautifulsoup4
    pip3 install pyquery
    qunzhudeMacBook-Air:bin qunzhupu$ python3
    Python 3.7.0 (default, Jun 29 2018, 20:13:13) 
    [Clang 9.1.0 (clang-902.0.39.2)] on darwin
    Type "help", "copyright", "credits" or "license" for more information.
    >>> from pyquery import PyQuery as pq
    >>> dec =pq('<html></html>')
    >>> dec =pq('<html>Hello</html>')
    >>> result = dec('html').text()
    >>> result
    'Hello'
    >>> 
    

    pymysql连接Mysql

    qunzhudeMacBook-Air:bin qunzhupu$ python3
    Python 3.7.0 (default, Jun 29 2018, 20:13:13) 
    [Clang 9.1.0 (clang-902.0.39.2)] on darwin
    Type "help", "copyright", "credits" or "license" for more information.
    >>> import pymysql
    >>> conn = pymysql.connect(host='123.206.204.80',user='puqunzhu',password='123456',port=3306,db='puqunzhu')
    >>> cursor = conn.cursor()
    >>> cursor.execute('select * from article')
    1
    

    pymongo连接mongoDB

    pip3 install pymongo
    qunzhudeMacBook-Air:bin qunzhupu$ python3
    Python 3.7.0 (default, Jun 29 2018, 20:13:13) 
    [Clang 9.1.0 (clang-902.0.39.2)] on darwin
    Type "help", "copyright", "credits" or "license" for more information.
    >>> import pymongo
    >>> client = pymongo.MongoClient('localhost')
    >>> db = client['newtestdb']
    >>> db['table'].insert({'name':'Bob'})
    __main__:1: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.
    ObjectId('5bc5b059e18a3620cc46bdb4')
    >>> db['table'].find_one({'name':'Bob'})
    {'_id': ObjectId('5bc5b059e18a3620cc46bdb4'), 'name': 'Bob'}
    

    redis连接redis

    pip3 install redis
    qunzhudeMacBook-Air:bin qunzhupu$ python3
    Python 3.7.0 (default, Jun 29 2018, 20:13:13) 
    [Clang 9.1.0 (clang-902.0.39.2)] on darwin
    Type "help", "copyright", "credits" or "license" for more information.
    >>> import redis
    >>> r = redis.Redis('localhost',6379)
    >>> r.set('name','puqunzhu')
    True
    >>> r.get('name')
    b'puqunzhu'
    

    flask代理获取

    pip3 install flask
    

    django

    pip3 install django
    

    jupyter

    pip3 install jupyter
    qunzhudeMacBook-Air:bin qunzhupu$ jupyter notebook
    
    大道理谁都懂,鸡汤也听过,可我们为什么还是过不好这一生。
  • 相关阅读:
    GitHub 的企业版
    我的Tag列表
    .net开发者对android开发一周的学习体会
    Ajax简单聊天B/S
    C#设计模式——享元模式(Flyweight Pattern)
    mongodb的sharding架构搭建
    Java设计模式
    LMAX架构
    Winform开发的常用类库
    C#设置本地网络(DNS、网关、子网掩码、IP)
  • 原文地址:https://www.cnblogs.com/puqunzhu/p/9799816.html
Copyright © 2011-2022 走看看