mitmproxy v0.18.2版本Python script示例

 

---

记录请求日志到MongoDB

# 记录请求日志到MongoDB
import pymongo
from datetime import datetime
from mitmproxy import ctx

# 连接MongoDB
client = pymongo.MongoClient()
db = client['mitmproxy']
collection = db['logs']

def request(flow):
    ctx.log.info("-->" + flow.request.url)
    #https://mitmproxy.readthedocs.io/en/v2.0.2/_modules/mitmproxy/net/http/request.html
    collection.insert({'url': flow.request.url,
                    'method': flow.request.method,
                    'headers': flow.request.headers,
                    'form': flow.request.urlencoded_form or flow.request.multipart_form,
                    'datetime': datetime.now()})

# 记录请求日志到MongoDB
import pymongo
from datetime import datetime
from mitmproxy import ctx

# 连接MongoDB
client = pymongo.MongoClient()
db = client['mitmproxy']
collection = db['logs']

def request(flow):
    ctx.log.info("-->" + flow.request.url)
    #https://mitmproxy.readthedocs.io/en/v2.0.2/_modules/mitmproxy/net/http/request.html
    collection.insert({'url': flow.request.url, 
                    'method': flow.request.method,
                    'headers': flow.request.headers, 
                    'form': flow.request.urlencoded_form or flow.request.multipart_form,
                    'datetime': datetime.now()})

拦截（屏蔽）某个请求

# 拦截百度https://www.baidu.com/link?url=，返回404
import re
from mitmproxy import ctx
from mitmproxy.models import HTTPResponse

def request(flow):
    """修改请求
    """
    if re.compile(r'/link?url=.*').search(flow.request.url):
        ctx.log.info(" --->" + flow.request.url)
        # 返回404
        flow.response = HTTPResponse.make(404)

# 拦截百度https://www.baidu.com/link?url=，返回404
import re
from mitmproxy import ctx
from mitmproxy.models import HTTPResponse
    
def request(flow):
    """修改请求
    """
    if re.compile(r'/link?url=.*').search(flow.request.url):
        ctx.log.info(" --->" + flow.request.url)
        # 返回404
        flow.response = HTTPResponse.make(404)

重写一个域名

# 将www.baidu.com域名改写为www.site-digger.com
from mitmproxy import ctx

def request(flow):
    if flow.request.pretty_host.endswith('www.baidu.com'):
        flow.request.host = 'www.site-digger.com'
        flow.request.scheme = 'http'
        flow.request.port = 80
        ctx.log.info(" --->" + flow.request.url)

# 将www.baidu.com域名改写为www.site-digger.com
from mitmproxy import ctx

def request(flow): 
    if flow.request.pretty_host.endswith('www.baidu.com'): 
        flow.request.host = 'www.site-digger.com' 
        flow.request.scheme = 'http' 
        flow.request.port = 80 
        ctx.log.info(" --->" + flow.request.url) 

修改应答数据

# 将百度首页的logo替换掉
import re
from mitmproxy import ctx

def response(flow):
    if flow.request.pretty_host.endswith('www.baidu.com') and flow.request.path.endswith('/'):
        flow.response.text = re.sub(r'//www.baidu.com/img/bd_logo[^"]+', 'http://www.site-digger.com/images/logo.png', flow.response.text)

# 将百度首页的logo替换掉
import re
from mitmproxy import ctx
    
def response(flow):
    if flow.request.pretty_host.endswith('www.baidu.com') and flow.request.path.endswith('/'):
        flow.response.text = re.sub(r'//www.baidu.com/img/bd_logo[^"]+', 'http://www.site-digger.com/images/logo.png', flow.response.text)

特别说明：本文旨在技术交流，请勿将涉及的技术用于非法用途，否则一切后果自负。如果您觉得我们侵犯了您的合法权益，请联系我们予以处理。

☹ Disqus被Qiang了，之前所有的评论内容都看不到了。如果您有爬虫相关技术方面的问题，欢迎发到我们的问答平台：http://spider.site-digger.com/