zoukankan      html  css  js  c++  java
  • python 多进程读写文件

    import time
    from multiprocessing import Process, JoinableQueue, cpu_count
    import csv
    
    
    ####处理一条数据的方法
    def deal_line(line, writer, csv_file):
        writer.writerow((line[3], line[1]))
        csv_file.flush()#重点,在多进程中写文件需要尽快刷新,否则可能会导致数据丢失
    
    ####消费者模型
    def consumer(queue, writer, csv_file):
        while True:
            line = queue.get()
            deal_line(line, writer, csv_file)
            queue.task_done()
    ####生产者模型
    def producer(queue):
        with open('test.txt', 'r') as f:
            for line in f:
                queue.put(line)
    
    ####启动N个生产者N个消费者模型
    def main():
        with open('t1.csv', 'w+') as csv_file:
            writer = csv.writer(csv_file)
            queue = JoinableQueue(8)#可限制队列长度
            pc = Process(target=producer, args=(queue,))
            pc.start()
    
            #多消费者
            for _ in range(cpu_count()):
                c1 = Process(target=consumer, args=(queue, writer, csv_file))
                c1.daemon = True
                c1.start()
            pc.join()#等待生产者进程全部生成完毕
            queue.join()# 等待所有数据全部处理完毕
    
            # 当某些些进程是死循环时可强制终止
            # pc.terminate()
    
    if __name__ == '__main__':
    
        now = lambda: time.time()
        start = now()
        main()
        print("Time:", now() - start)
    

      

  • 相关阅读:
    docker部署springboot应用
    docker部署vue前端
    docker安装mysql5.7 数据挂载
    docker 部署nestjs应用
    unbuntu16.04安装geoserver运行环境
    windows server2012 搭建.netcore+nginx+nssm运行环境
    nginx部署多个网站
    vscode搭建springboot开发环境
    软工课程总结
    Beta答辩总结
  • 原文地址:https://www.cnblogs.com/mahailuo/p/11460739.html
Copyright © 2011-2022 走看看