SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 62332 and this is thread id 53032
sqlite支持的并发访问数??
import xlrd import time import sys import os import requests import sqlite3 import threading curPath = os.path.abspath(os.path.dirname(__file__)) rootPath = os.path.split(curPath)[0] sys.path.append(rootPath) MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST' db = 'py_bdspider_status.db' def db_init_key_table(): conn = sqlite3.connect(db) c = conn.cursor() sql = 'DELETE FROM baidu_map_key_used' c.execute(sql) conn.commit() pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt') with open(pcity_file, 'r', encoding='utf-8') as pf: c_ = 0 for i in pf: if len(i) < 4: continue author, key = i.replace(' ', '').split(' ') localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime()) sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % ( author, key, localtime_, 0) c.execute(sql) conn.commit() conn.close() # db_init_key_table() def db_get_one_effective(): conn = sqlite3.connect(db) c = conn.cursor() sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES) res = c.execute(sql).fetchone() if res is None: return DB_KEY_EXHAUST else: return res[0] conn.close def db_update_one_today_used(key): conn = sqlite3.connect(db) c = conn.cursor() localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime()) sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % ( localtime_, key) c.execute(sql) conn.commit() conn.close() dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception' requested_file_list = [] requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % ( curPath, dir_exception) requested_file_dir = os.listdir(requested_file_dir_str) def chk_if_requested_file(): for f in requested_file_dir: to_in = f.split('.txt')[0] if to_in not in requested_file_list: requested_file_list.append(to_in) def write_requested_res(request_name, str_, type_='.txt'): fname = '%s%s%s' % (requested_file_dir_str, request_name, type_) with open(fname, 'w', encoding='utf-8') as ft: ft.write(str_) print('ok', threading.get_ident(), request_name) def write_requested_exception_res(request_name, str_, type_='.txt'): fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_) with open(fname, 'w', encoding='utf-8') as ft: ft.write(str_) fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821' def fun_(): fname_open = '%s\%s' % (curPath, fname_source) FEXCEL = '%s%s' % (fname_open, '.xlsx') data = xlrd.open_workbook(FEXCEL) table = data.sheets()[0] nrows, ncols = table.nrows, table.ncols # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下®ion=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1 base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY®ion=R-CITY&city_limit=true&output=json&ak=R-AK' for i in range(1, nrows): l = table.row_values(i) dbid, area_code, name_, request_name, type_, city, district, addr, street = l request_name_chk = '%s%s%s' % (city, district, request_name) chk_if_requested_file() if request_name_chk in requested_file_list: continue ak = db_get_one_effective() if ak == DB_KEY_EXHAUST: print(DB_KEY_EXHAUST) break else: url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak) try: bd_res_json_str = requests.get(url_).text db_update_one_today_used(ak) write_requested_res(request_name_chk, bd_res_json_str) except Exception: bd_res_json_str = '请求百度-异常' write_requested_exception_res(request_name_chk, bd_res_json_str) print(bd_res_json_str) class MyThread(threading.Thread): def __init__(self, func): threading.Thread.__init__(self) self.func = func def run(self): self.func() thread_sum = 16 def main(): threads_list = [] for nloop in range(0, thread_sum, 1): thread_instance = MyThread(fun_) threads_list.append(thread_instance) for t in threads_list: t.setDaemon = False t.start() for t in threads_list: t.join() if __name__ == '__main__': main()
import xlrd import time import sys import os import requests import sqlite3 import threading curPath = os.path.abspath(os.path.dirname(__file__)) rootPath = os.path.split(curPath)[0] sys.path.append(rootPath) MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST' db = 'py_bdspider_status.db' def db_init_key_table(): conn = sqlite3.connect(db) c = conn.cursor() sql = 'DELETE FROM baidu_map_key_used' c.execute(sql) conn.commit() pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt') with open(pcity_file, 'r', encoding='utf-8') as pf: c_ = 0 for i in pf: if len(i) < 4: continue author, key = i.replace(' ', '').split(' ') localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime()) sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % ( author, key, localtime_, 0) c.execute(sql) conn.commit() conn.close() # db_init_key_table() def db_get_one_effective(): conn = sqlite3.connect(db) c = conn.cursor() sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES) res = c.execute(sql).fetchone() if res is None: return DB_KEY_EXHAUST else: return res[0] conn.close def db_update_one_today_used(key): conn = sqlite3.connect(db) c = conn.cursor() localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime()) sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % ( localtime_, key) c.execute(sql) conn.commit() conn.close() dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception' requested_file_list = [] requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % ( curPath, dir_exception) requested_file_dir = os.listdir(requested_file_dir_str) def chk_if_requested_file(): for f in requested_file_dir: to_in = f.split('.txt')[0] if to_in not in requested_file_list: requested_file_list.append(to_in) def write_requested_res(request_name, str_, type_='.txt'): fname = '%s%s%s' % (requested_file_dir_str, request_name, type_) with open(fname, 'w', encoding='utf-8') as ft: ft.write(str_) print('ok', threading.get_ident(), request_name) def write_requested_exception_res(request_name, str_, type_='.txt'): fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_) with open(fname, 'w', encoding='utf-8') as ft: ft.write(str_) fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821' def fun_(): fname_open = '%s\%s' % (curPath, fname_source) FEXCEL = '%s%s' % (fname_open, '.xlsx') data = xlrd.open_workbook(FEXCEL) table = data.sheets()[0] nrows, ncols = table.nrows, table.ncols # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下®ion=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1 base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY®ion=R-CITY&city_limit=true&output=json&ak=R-AK' i_counter = 0 for i in range(1, nrows): l = table.row_values(i) i_counter += 1 dbid, area_code, name_, request_name, type_, city, district, addr, street = l request_name_chk = '%s%s%s' % (city, district, request_name) chk_if_requested_file() if request_name_chk in requested_file_list: continue ak = db_get_one_effective() if ak == DB_KEY_EXHAUST: print(DB_KEY_EXHAUST) break else: url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak) try: bd_res_json_str = requests.get(url_).text db_update_one_today_used(ak) write_requested_res(request_name_chk, bd_res_json_str) except Exception: bd_res_json_str = '请求百度-异常' write_requested_exception_res(request_name_chk, bd_res_json_str) print(bd_res_json_str) class MyThread(threading.Thread): def __init__(self, func): threading.Thread.__init__(self) self.func = func def run(self): self.func() thread_sum = 100 def main(): threads_list = [] for nloop in range(0, thread_sum, 1): thread_instance = MyThread(fun_) threads_list.append(thread_instance) for t in threads_list: t.setDaemon = False t.start() for t in threads_list: t.join() if __name__ == '__main__': main()
import xlrd import time import sys import os import requests import sqlite3 import threading curPath = os.path.abspath(os.path.dirname(__file__)) rootPath = os.path.split(curPath)[0] sys.path.append(rootPath) db = 'py_bdspider_status.db' conn = sqlite3.connect(db) c = conn.cursor() MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST' def db_init_key_table(): sql = 'DELETE FROM baidu_map_key_used' c.execute(sql) conn.commit() pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt') with open(pcity_file, 'r', encoding='utf-8') as pf: c_ = 0 for i in pf: if len(i) < 4: continue author, key = i.replace(' ', '').split(' ') localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime()) sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % ( author, key, localtime_, 0) c.execute(sql) conn.commit() # db_init_key_table() def db_get_one_effective(): sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES) res = c.execute(sql).fetchone() if res is None: return DB_KEY_EXHAUST else: return res[0] def db_update_one_today_used(key): localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime()) sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % ( localtime_, key) c.execute(sql) conn.commit() dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception' requested_file_list = [] requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % ( curPath, dir_exception) requested_file_dir = os.listdir(requested_file_dir_str) def chk_if_requested_file(): for f in requested_file_dir: to_in = f.split('.txt')[0] if to_in not in requested_file_list: requested_file_list.append(to_in) def write_requested_res(request_name, str_, type_='.txt'): fname = '%s%s%s' % (requested_file_dir_str, request_name, type_) with open(fname, 'w', encoding='utf-8') as ft: ft.write(str_) print('ok', threading.get_ident(), request_name) def write_requested_exception_res(request_name, str_, type_='.txt'): fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_) with open(fname, 'w', encoding='utf-8') as ft: ft.write(str_) fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821' def fun_(): fname_open = '%s\%s' % (curPath, fname_source) FEXCEL = '%s%s' % (fname_open, '.xlsx') data = xlrd.open_workbook(FEXCEL) table = data.sheets()[0] nrows, ncols = table.nrows, table.ncols # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下®ion=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1 base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY®ion=R-CITY&city_limit=true&output=json&ak=R-AK' i_counter = 0 for i in range(1, nrows): l = table.row_values(i) i_counter += 1 print(i_counter) if i_counter > 60: break dbid, area_code, name_, request_name, type_, city, district, addr, street = l request_name_chk = '%s%s%s' % (city, district, request_name) chk_if_requested_file() if request_name_chk in requested_file_list: continue ak = db_get_one_effective() if ak == DB_KEY_EXHAUST: print(DB_KEY_EXHAUST) break else: url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak) print(url_) try: bd_res_json_str = requests.get(url_).text db_update_one_today_used(ak) write_requested_res(request_name_chk, bd_res_json_str) except Exception: bd_res_json_str = '请求百度-异常' write_requested_exception_res(request_name_chk, bd_res_json_str) print(bd_res_json_str) class MyThread(threading.Thread): def __init__(self, func): threading.Thread.__init__(self) self.func = func def run(self): self.func() thread_sum = 4 def main(): threads_list = [] for nloop in range(0, thread_sum, 1): thread_instance = MyThread(fun_) threads_list.append(thread_instance) for t in threads_list: t.setDaemon = False t.start() for t in threads_list: t.join() conn.close() if __name__ == '__main__': main()