1 from scrapy import log
2 import pymysql
3 import pymysql.cursors
4 import codecs
5 from twisted.enterprise import adbapi
6
7 class MysqlTwistedPipeline(object):
8
9 @classmethod
10 def from_settings(cls, settings):
11 dbargs = dict(
12 host=settings['MYSQL_HOST'],
13 db=settings['MYSQL_DBNAME'],
14 user=settings['MYSQL_USER'],
15 passwd=settings['MYSQL_PASSWD'],
16 port=settings['MYSQL_PORT'],
17 charset='utf8',
18 cursorclass=pymysql.cursors.DictCursor,
19 use_unicode=True,
20 )
21 dbpool = adbapi.ConnectionPool('pymysql', **dbargs)
22 return cls(dbpool)
23
24
25 def __init__(self,dbpool):
26 self.dbpool=dbpool
27
28 #pipeline默认调用
29 def process_item(self, item, spider):
30 d=self.dbpool.runInteraction(self._conditional_insert, item, spider)#调用插入的方法
31 log.msg("-------------------连接好了-------------------")
32 d.addErrback(self._handle_error,item,spider)#调用异常处理方法
33 d.addBoth(lambda _: item)
34 return d
35
36 def _conditional_insert(self, conn, item, spider):
37 log.msg("-------------------打印-------------------")
38
39 conn.execute("insert into MovieInfo (poster, name, director, actor, tag_char, score, critics_num,) values(%s, %s, %s,%s,%s,%s,%s,)",
40 (item['cover'], item['title'], item['region'], item['years'], item['types'], item['score'], item['comment_num'],))
41 log.msg("-------------------一轮循环完毕-------------------")
42 def _handle_error(self, failue, item, spider):
43 print(failue)
