zoukankan      html  css  js  c++  java
  • python引入线程池进行多线程操作

    import sys
    from glob import glob
    from os import path
    import random
    from redis import StrictRedis
    from BICDetection import biccheck 
    from concurrent.futures import ProcessPoolExecutor  #  引入进程池 然后将进程处理过的结果存入大Redis中,这样可以在多台服务器上直接跑数据。
    
    def get_direct_fps(fp):
        with open(fp) as fd:
            fps = [path.join(root_3421, i[:8], i.split()[0]) for i in fd]
            fps.sort()
        return fps
    
    def get_fps(fp):
        with open(fp) as fd: 
            return [i.strip() for i in fd]
    
    folder = 'PWD'
    fps = glob(path.join(folder, '*.wav'))
    
    redis_conn = StrictRedis(host="IP", db=0)
    
    def bic_check_wrapper(args):
        fp, idx, label = args
        if not path.exists(fp):
            print(fp)
            return
        filename = path.basename(fp)
        if redis_conn.hget(label, filename) is not None:
            return
        r = biccheck([fp])
        flag = r['flag'].iloc[0]
        redis_conn.hset(label, filename, flag)
        return flag
    
    def cu_flag_detect(suffix, nproc=30, select_num=None):
        passed_fps = get_direct_fps('cu.log')
        new_args = [(fp, idx, 'passed_{}'.format(suffix)) for idx, fp in enumerate(passed_fps)]
        unpassed_fps = get_fps('cu_71.log')
        new_args += [(fp, idx, 'unpassed_{}'.format(suffix)) for idx, fp in enumerate(unpassed_fps)]
        random.seed(1)
        random.shuffle(new_args)
        if select_num is None:
            selected = new_args
        with ProcessPoolExecutor(int(nproc)) as executor:
            executor.map(bic_check_wrapper, selected)
    
    
    关注公众号 海量干货等你
  • 相关阅读:
    互联网实习笔记之shell笔记
    互联网实习笔记之30天总结
    记被论文排版虐的一天
    大论文排版技巧
    Matlab2014a使用VS2015混合编译
    腾讯云服务器创建swap空间
    使用vs2015开发linux:Ubuntu程序
    使用VS2015远程GDB调试
    nullptr、NULL、null和0
    《将博客搬至CSDN》
  • 原文地址:https://www.cnblogs.com/sowhat1412/p/12734312.html
Copyright © 2011-2022 走看看