zoukankan      html  css  js  c++  java
  • SQLite支持的并发访问数

     SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 62332 and this is thread id 53032

    sqlite支持的并发访问数??

    import xlrd
    import time
    import sys
    import os
    import requests
    import sqlite3
    import threading
    
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = os.path.split(curPath)[0]
    sys.path.append(rootPath)
    
    MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST'
    
    db = 'py_bdspider_status.db'
    
    
    def db_init_key_table():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'DELETE  FROM  baidu_map_key_used'
        c.execute(sql)
        conn.commit()
        pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt')
        with open(pcity_file, 'r', encoding='utf-8') as pf:
            c_ = 0
            for i in pf:
                if len(i) < 4:
                    continue
                author, key = i.replace('
    ', '').split('	')
                localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
                sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (
                    author, key, localtime_, 0)
                c.execute(sql)
        conn.commit()
        conn.close()
    
    
    # db_init_key_table()
    
    def db_get_one_effective():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)
        res = c.execute(sql).fetchone()
        if res is None:
            return DB_KEY_EXHAUST
        else:
            return res[0]
        conn.close
    
    
    def db_update_one_today_used(key):
        conn = sqlite3.connect(db)
        c = conn.cursor()
        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
        sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (
            localtime_, key)
        c.execute(sql)
        conn.commit()
        conn.close()
    
    
    dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'
    requested_file_list = []
    requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % (
        curPath, dir_exception)
    requested_file_dir = os.listdir(requested_file_dir_str)
    
    
    def chk_if_requested_file():
        for f in requested_file_dir:
            to_in = f.split('.txt')[0]
            if to_in not in requested_file_list:
                requested_file_list.append(to_in)
    
    
    def write_requested_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
        print('ok', threading.get_ident(), request_name)
    
    
    def write_requested_exception_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
    
    
    fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'
    
    
    def fun_():
        fname_open = '%s\%s' % (curPath, fname_source)
        FEXCEL = '%s%s' % (fname_open, '.xlsx')
        data = xlrd.open_workbook(FEXCEL)
        table = data.sheets()[0]
        nrows, ncols = table.nrows, table.ncols
        # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1
        base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'
        for i in range(1, nrows):
            l = table.row_values(i)
            dbid, area_code, name_, request_name, type_, city, district, addr, street = l
            request_name_chk = '%s%s%s' % (city, district, request_name)
            chk_if_requested_file()
            if request_name_chk in requested_file_list:
                continue
            ak = db_get_one_effective()
            if ak == DB_KEY_EXHAUST:
                print(DB_KEY_EXHAUST)
                break
            else:
                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)
            try:
                bd_res_json_str = requests.get(url_).text
                db_update_one_today_used(ak)
                write_requested_res(request_name_chk, bd_res_json_str)
            except Exception:
                bd_res_json_str = '请求百度-异常'
                write_requested_exception_res(request_name_chk, bd_res_json_str)
                print(bd_res_json_str)
    
    
    class MyThread(threading.Thread):
        def __init__(self, func):
            threading.Thread.__init__(self)
            self.func = func
    
        def run(self):
            self.func()
    
    
    thread_sum = 16
    
    
    def main():
        threads_list = []
        for nloop in range(0, thread_sum, 1):
            thread_instance = MyThread(fun_)
            threads_list.append(thread_instance)
        for t in threads_list:
            t.setDaemon = False
            t.start()
        for t in threads_list:
            t.join()
    
    
    if __name__ == '__main__':
        main()
    

      

    import xlrd
    import time
    import sys
    import os
    import requests
    import sqlite3
    import threading
    
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = os.path.split(curPath)[0]
    sys.path.append(rootPath)
    
    MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST'
    
    db = 'py_bdspider_status.db'
    
    
    def db_init_key_table():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'DELETE  FROM  baidu_map_key_used'
        c.execute(sql)
        conn.commit()
        pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt')
        with open(pcity_file, 'r', encoding='utf-8') as pf:
            c_ = 0
            for i in pf:
                if len(i) < 4:
                    continue
                author, key = i.replace('
    ', '').split('	')
                localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
                sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (
                    author, key, localtime_, 0)
                c.execute(sql)
        conn.commit()
        conn.close()
    
    
    # db_init_key_table()
    
    def db_get_one_effective():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)
        res = c.execute(sql).fetchone()
        if res is None:
            return DB_KEY_EXHAUST
        else:
            return res[0]
        conn.close
    
    
    def db_update_one_today_used(key):
        conn = sqlite3.connect(db)
        c = conn.cursor()
        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
        sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (
            localtime_, key)
        c.execute(sql)
        conn.commit()
        conn.close()
    
    
    dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'
    requested_file_list = []
    requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % (
        curPath, dir_exception)
    requested_file_dir = os.listdir(requested_file_dir_str)
    
    
    def chk_if_requested_file():
        for f in requested_file_dir:
            to_in = f.split('.txt')[0]
            if to_in not in requested_file_list:
                requested_file_list.append(to_in)
    
    
    def write_requested_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
        print('ok', threading.get_ident(), request_name)
    
    
    def write_requested_exception_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
    
    
    fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'
    
    
    def fun_():
        fname_open = '%s\%s' % (curPath, fname_source)
        FEXCEL = '%s%s' % (fname_open, '.xlsx')
        data = xlrd.open_workbook(FEXCEL)
        table = data.sheets()[0]
        nrows, ncols = table.nrows, table.ncols
        # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1
        base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'
        i_counter = 0
        for i in range(1, nrows):
            l = table.row_values(i)
            i_counter += 1
            dbid, area_code, name_, request_name, type_, city, district, addr, street = l
            request_name_chk = '%s%s%s' % (city, district, request_name)
            chk_if_requested_file()
            if request_name_chk in requested_file_list:
                continue
            ak = db_get_one_effective()
            if ak == DB_KEY_EXHAUST:
                print(DB_KEY_EXHAUST)
                break
            else:
                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)
            try:
                bd_res_json_str = requests.get(url_).text
                db_update_one_today_used(ak)
                write_requested_res(request_name_chk, bd_res_json_str)
            except Exception:
                bd_res_json_str = '请求百度-异常'
                write_requested_exception_res(request_name_chk, bd_res_json_str)
                print(bd_res_json_str)
    
    
    class MyThread(threading.Thread):
        def __init__(self, func):
            threading.Thread.__init__(self)
            self.func = func
    
        def run(self):
            self.func()
    
    
    thread_sum = 100
    
    
    def main():
        threads_list = []
        for nloop in range(0, thread_sum, 1):
            thread_instance = MyThread(fun_)
            threads_list.append(thread_instance)
        for t in threads_list:
            t.setDaemon = False
            t.start()
        for t in threads_list:
            t.join()
    
    
    if __name__ == '__main__':
        main()
    

      

    import xlrd
    import time
    import sys
    import os
    import requests
    import sqlite3
    import threading
    
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = os.path.split(curPath)[0]
    sys.path.append(rootPath)
    
    db = 'py_bdspider_status.db'
    conn = sqlite3.connect(db)
    c = conn.cursor()
    MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST'
    
    
    def db_init_key_table():
        sql = 'DELETE  FROM  baidu_map_key_used'
        c.execute(sql)
        conn.commit()
        pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt')
        with open(pcity_file, 'r', encoding='utf-8') as pf:
            c_ = 0
            for i in pf:
                if len(i) < 4:
                    continue
                author, key = i.replace('
    ', '').split('	')
                localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
                sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (
                    author, key, localtime_, 0)
                c.execute(sql)
        conn.commit()
    
    
    # db_init_key_table()
    
    def db_get_one_effective():
        sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)
        res = c.execute(sql).fetchone()
        if res is None:
            return DB_KEY_EXHAUST
        else:
            return res[0]
    
    
    def db_update_one_today_used(key):
        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
        sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (
            localtime_, key)
        c.execute(sql)
        conn.commit()
    
    
    dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'
    requested_file_list = []
    requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % (
        curPath, dir_exception)
    requested_file_dir = os.listdir(requested_file_dir_str)
    
    
    def chk_if_requested_file():
        for f in requested_file_dir:
            to_in = f.split('.txt')[0]
            if to_in not in requested_file_list:
                requested_file_list.append(to_in)
    
    
    def write_requested_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
        print('ok', threading.get_ident(), request_name)
    
    
    def write_requested_exception_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
    
    
    fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'
    
    
    def fun_():
        fname_open = '%s\%s' % (curPath, fname_source)
        FEXCEL = '%s%s' % (fname_open, '.xlsx')
        data = xlrd.open_workbook(FEXCEL)
        table = data.sheets()[0]
        nrows, ncols = table.nrows, table.ncols
        # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1
        base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'
        i_counter = 0
        for i in range(1, nrows):
            l = table.row_values(i)
            i_counter += 1
            print(i_counter)
            if i_counter > 60:
                break
            dbid, area_code, name_, request_name, type_, city, district, addr, street = l
            request_name_chk = '%s%s%s' % (city, district, request_name)
            chk_if_requested_file()
            if request_name_chk in requested_file_list:
                continue
            ak = db_get_one_effective()
            if ak == DB_KEY_EXHAUST:
                print(DB_KEY_EXHAUST)
                break
            else:
                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)
                print(url_)
            try:
                bd_res_json_str = requests.get(url_).text
                db_update_one_today_used(ak)
                write_requested_res(request_name_chk, bd_res_json_str)
            except Exception:
                bd_res_json_str = '请求百度-异常'
                write_requested_exception_res(request_name_chk, bd_res_json_str)
                print(bd_res_json_str)
    
    
    
    class MyThread(threading.Thread):
        def __init__(self, func):
            threading.Thread.__init__(self)
            self.func = func
    
        def run(self):
            self.func()
    
    thread_sum = 4
    
    
    def main():
        threads_list = []
        for nloop in range(0, thread_sum, 1):
            thread_instance = MyThread(fun_)
            threads_list.append(thread_instance)
        for t in threads_list:
            t.setDaemon = False
            t.start()
        for t in threads_list:
            t.join()
    
        conn.close()
    
    
    if __name__ == '__main__':
        main()
    

      

  • 相关阅读:
    如何提高码农产量,基于ASP.NET MVC的敏捷开发框架开发随笔一
    ASP.NET MVC快速开发框架清新简洁界面设计,有兴趣可以模仿参考
    大湿教我写程序(3)之自动补全(屌丝没有春天)篇
    大湿教我写程序(2)之走向AV之路
    大湿教我写程序(1)之菜单导航篇
    打造一套UI与后台并重.net通用权限管理系统
    如何提高码农产量,基于ASP.NET MVC的敏捷开发框架之自定义表单开发随笔四
    如何提高码农产量,基于ASP.NET MVC的敏捷开发框架之工作流开发随笔三
    摆平客户的需求变更之表单扩展属性
    快速开发框架(FDMS)新增1000个对外接口都不须要手写一行代码
  • 原文地址:https://www.cnblogs.com/rsapaper/p/7449247.html
Copyright © 2011-2022 走看看