zoukankan      html  css  js  c++  java
  • SQLite支持的并发访问数

     SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 62332 and this is thread id 53032

    sqlite支持的并发访问数??

    import xlrd
    import time
    import sys
    import os
    import requests
    import sqlite3
    import threading
    
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = os.path.split(curPath)[0]
    sys.path.append(rootPath)
    
    MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST'
    
    db = 'py_bdspider_status.db'
    
    
    def db_init_key_table():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'DELETE  FROM  baidu_map_key_used'
        c.execute(sql)
        conn.commit()
        pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt')
        with open(pcity_file, 'r', encoding='utf-8') as pf:
            c_ = 0
            for i in pf:
                if len(i) < 4:
                    continue
                author, key = i.replace('
    ', '').split('	')
                localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
                sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (
                    author, key, localtime_, 0)
                c.execute(sql)
        conn.commit()
        conn.close()
    
    
    # db_init_key_table()
    
    def db_get_one_effective():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)
        res = c.execute(sql).fetchone()
        if res is None:
            return DB_KEY_EXHAUST
        else:
            return res[0]
        conn.close
    
    
    def db_update_one_today_used(key):
        conn = sqlite3.connect(db)
        c = conn.cursor()
        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
        sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (
            localtime_, key)
        c.execute(sql)
        conn.commit()
        conn.close()
    
    
    dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'
    requested_file_list = []
    requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % (
        curPath, dir_exception)
    requested_file_dir = os.listdir(requested_file_dir_str)
    
    
    def chk_if_requested_file():
        for f in requested_file_dir:
            to_in = f.split('.txt')[0]
            if to_in not in requested_file_list:
                requested_file_list.append(to_in)
    
    
    def write_requested_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
        print('ok', threading.get_ident(), request_name)
    
    
    def write_requested_exception_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
    
    
    fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'
    
    
    def fun_():
        fname_open = '%s\%s' % (curPath, fname_source)
        FEXCEL = '%s%s' % (fname_open, '.xlsx')
        data = xlrd.open_workbook(FEXCEL)
        table = data.sheets()[0]
        nrows, ncols = table.nrows, table.ncols
        # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1
        base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'
        for i in range(1, nrows):
            l = table.row_values(i)
            dbid, area_code, name_, request_name, type_, city, district, addr, street = l
            request_name_chk = '%s%s%s' % (city, district, request_name)
            chk_if_requested_file()
            if request_name_chk in requested_file_list:
                continue
            ak = db_get_one_effective()
            if ak == DB_KEY_EXHAUST:
                print(DB_KEY_EXHAUST)
                break
            else:
                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)
            try:
                bd_res_json_str = requests.get(url_).text
                db_update_one_today_used(ak)
                write_requested_res(request_name_chk, bd_res_json_str)
            except Exception:
                bd_res_json_str = '请求百度-异常'
                write_requested_exception_res(request_name_chk, bd_res_json_str)
                print(bd_res_json_str)
    
    
    class MyThread(threading.Thread):
        def __init__(self, func):
            threading.Thread.__init__(self)
            self.func = func
    
        def run(self):
            self.func()
    
    
    thread_sum = 16
    
    
    def main():
        threads_list = []
        for nloop in range(0, thread_sum, 1):
            thread_instance = MyThread(fun_)
            threads_list.append(thread_instance)
        for t in threads_list:
            t.setDaemon = False
            t.start()
        for t in threads_list:
            t.join()
    
    
    if __name__ == '__main__':
        main()
    

      

    import xlrd
    import time
    import sys
    import os
    import requests
    import sqlite3
    import threading
    
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = os.path.split(curPath)[0]
    sys.path.append(rootPath)
    
    MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST'
    
    db = 'py_bdspider_status.db'
    
    
    def db_init_key_table():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'DELETE  FROM  baidu_map_key_used'
        c.execute(sql)
        conn.commit()
        pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt')
        with open(pcity_file, 'r', encoding='utf-8') as pf:
            c_ = 0
            for i in pf:
                if len(i) < 4:
                    continue
                author, key = i.replace('
    ', '').split('	')
                localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
                sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (
                    author, key, localtime_, 0)
                c.execute(sql)
        conn.commit()
        conn.close()
    
    
    # db_init_key_table()
    
    def db_get_one_effective():
        conn = sqlite3.connect(db)
        c = conn.cursor()
        sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)
        res = c.execute(sql).fetchone()
        if res is None:
            return DB_KEY_EXHAUST
        else:
            return res[0]
        conn.close
    
    
    def db_update_one_today_used(key):
        conn = sqlite3.connect(db)
        c = conn.cursor()
        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
        sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (
            localtime_, key)
        c.execute(sql)
        conn.commit()
        conn.close()
    
    
    dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'
    requested_file_list = []
    requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % (
        curPath, dir_exception)
    requested_file_dir = os.listdir(requested_file_dir_str)
    
    
    def chk_if_requested_file():
        for f in requested_file_dir:
            to_in = f.split('.txt')[0]
            if to_in not in requested_file_list:
                requested_file_list.append(to_in)
    
    
    def write_requested_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
        print('ok', threading.get_ident(), request_name)
    
    
    def write_requested_exception_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
    
    
    fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'
    
    
    def fun_():
        fname_open = '%s\%s' % (curPath, fname_source)
        FEXCEL = '%s%s' % (fname_open, '.xlsx')
        data = xlrd.open_workbook(FEXCEL)
        table = data.sheets()[0]
        nrows, ncols = table.nrows, table.ncols
        # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1
        base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'
        i_counter = 0
        for i in range(1, nrows):
            l = table.row_values(i)
            i_counter += 1
            dbid, area_code, name_, request_name, type_, city, district, addr, street = l
            request_name_chk = '%s%s%s' % (city, district, request_name)
            chk_if_requested_file()
            if request_name_chk in requested_file_list:
                continue
            ak = db_get_one_effective()
            if ak == DB_KEY_EXHAUST:
                print(DB_KEY_EXHAUST)
                break
            else:
                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)
            try:
                bd_res_json_str = requests.get(url_).text
                db_update_one_today_used(ak)
                write_requested_res(request_name_chk, bd_res_json_str)
            except Exception:
                bd_res_json_str = '请求百度-异常'
                write_requested_exception_res(request_name_chk, bd_res_json_str)
                print(bd_res_json_str)
    
    
    class MyThread(threading.Thread):
        def __init__(self, func):
            threading.Thread.__init__(self)
            self.func = func
    
        def run(self):
            self.func()
    
    
    thread_sum = 100
    
    
    def main():
        threads_list = []
        for nloop in range(0, thread_sum, 1):
            thread_instance = MyThread(fun_)
            threads_list.append(thread_instance)
        for t in threads_list:
            t.setDaemon = False
            t.start()
        for t in threads_list:
            t.join()
    
    
    if __name__ == '__main__':
        main()
    

      

    import xlrd
    import time
    import sys
    import os
    import requests
    import sqlite3
    import threading
    
    curPath = os.path.abspath(os.path.dirname(__file__))
    rootPath = os.path.split(curPath)[0]
    sys.path.append(rootPath)
    
    db = 'py_bdspider_status.db'
    conn = sqlite3.connect(db)
    c = conn.cursor()
    MAX_USED_TIMES, overrun_str, DB_KEY_EXHAUST = 1980, '天配额超限,限制访问', 'DB_KEY_EXHAUST'
    
    
    def db_init_key_table():
        sql = 'DELETE  FROM  baidu_map_key_used'
        c.execute(sql)
        conn.commit()
        pcity_file = '%s\%s' % (curPath, 'bdmap_key.txt')
        with open(pcity_file, 'r', encoding='utf-8') as pf:
            c_ = 0
            for i in pf:
                if len(i) < 4:
                    continue
                author, key = i.replace('
    ', '').split('	')
                localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
                sql = 'INSERT INTO baidu_map_key_used (author,key,update_time,today_used) VALUES ("%s","%s","%s",%s) ' % (
                    author, key, localtime_, 0)
                c.execute(sql)
        conn.commit()
    
    
    # db_init_key_table()
    
    def db_get_one_effective():
        sql = 'SELECT key FROM baidu_map_key_used WHERE today_used<=%s ' % (MAX_USED_TIMES)
        res = c.execute(sql).fetchone()
        if res is None:
            return DB_KEY_EXHAUST
        else:
            return res[0]
    
    
    def db_update_one_today_used(key):
        localtime_ = time.strftime("%y%m%d%H%M%S", time.localtime())
        sql = 'UPDATE baidu_map_key_used SET today_used = today_used+1 ,update_time=%s WHERE key="%s" ' % (
            localtime_, key)
        c.execute(sql)
        conn.commit()
    
    
    dir_, dir_exception = 'baidu_map_uid', 'baidu_map_uid_exception'
    requested_file_list = []
    requested_file_dir_str, requested_file_dir_exception_str = '%s\%s\' % (curPath, dir_), '%s\%s\' % (
        curPath, dir_exception)
    requested_file_dir = os.listdir(requested_file_dir_str)
    
    
    def chk_if_requested_file():
        for f in requested_file_dir:
            to_in = f.split('.txt')[0]
            if to_in not in requested_file_list:
                requested_file_list.append(to_in)
    
    
    def write_requested_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
        print('ok', threading.get_ident(), request_name)
    
    
    def write_requested_exception_res(request_name, str_, type_='.txt'):
        fname = '%s%s%s' % (requested_file_dir_exception_str, request_name, type_)
        with open(fname, 'w', encoding='utf-8') as ft:
            ft.write(str_)
    
    
    fname_source = '官方上传任务.csv_py170829093808-BD_request_name-REDUCTION170829142821'
    
    
    def fun_():
        fname_open = '%s\%s' % (curPath, fname_source)
        FEXCEL = '%s%s' % (fname_open, '.xlsx')
        data = xlrd.open_workbook(FEXCEL)
        table = data.sheets()[0]
        nrows, ncols = table.nrows, table.ncols
        # http://api.map.baidu.com/place/v2/suggestion?query=瀛嘉天下&region=重庆市&city_limit=true&output=json&ak=oy2Q7IluhhwTGlz6l8pXYv6a0m6hXxr1
        base_url = 'http://api.map.baidu.com/place/v2/suggestion?query=R-QUERY&region=R-CITY&city_limit=true&output=json&ak=R-AK'
        i_counter = 0
        for i in range(1, nrows):
            l = table.row_values(i)
            i_counter += 1
            print(i_counter)
            if i_counter > 60:
                break
            dbid, area_code, name_, request_name, type_, city, district, addr, street = l
            request_name_chk = '%s%s%s' % (city, district, request_name)
            chk_if_requested_file()
            if request_name_chk in requested_file_list:
                continue
            ak = db_get_one_effective()
            if ak == DB_KEY_EXHAUST:
                print(DB_KEY_EXHAUST)
                break
            else:
                url_ = base_url.replace('R-QUERY', request_name).replace('R-CITY', city).replace('R-AK', ak)
                print(url_)
            try:
                bd_res_json_str = requests.get(url_).text
                db_update_one_today_used(ak)
                write_requested_res(request_name_chk, bd_res_json_str)
            except Exception:
                bd_res_json_str = '请求百度-异常'
                write_requested_exception_res(request_name_chk, bd_res_json_str)
                print(bd_res_json_str)
    
    
    
    class MyThread(threading.Thread):
        def __init__(self, func):
            threading.Thread.__init__(self)
            self.func = func
    
        def run(self):
            self.func()
    
    thread_sum = 4
    
    
    def main():
        threads_list = []
        for nloop in range(0, thread_sum, 1):
            thread_instance = MyThread(fun_)
            threads_list.append(thread_instance)
        for t in threads_list:
            t.setDaemon = False
            t.start()
        for t in threads_list:
            t.join()
    
        conn.close()
    
    
    if __name__ == '__main__':
        main()
    

      

  • 相关阅读:
    Windows多线程编程入门
    多字节字符与宽字符
    Linux静态库与动态库详解
    Linux下清理内存和Cache方法
    数据库设计范式
    mybatis面试问题
    Gson使用
    Linux 定时任务crontab使用
    Java-GC机制
    java内存模型
  • 原文地址:https://www.cnblogs.com/rsapaper/p/7449247.html
Copyright © 2011-2022 走看看