Python多线程实现进度条

Python多线程显示进度条的方式

使用python进行多线程任务时，有时需要获取及时进度来清晰的感知的任务进度，目前收集以下几种实用的方式。

使用 multiprocessing 进行记录

利用 callback来检测每个任务的完成，返回值为主函数的return。apply_async(subprocess, args=(i,), callback=log)

#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time    : 2018/5/23 14:15
# @Author  : zhouyuyao
# @File    : demon1.py
import multiprocessing
import os
import time
from datetime import datetime
from tqdm import tqdm

global flag
flag = 0
global max

def log(i):
    print(i)
    global flag
    flag =  flag + 1
    print(process_bar(flag, max))

def process_bar(num, total):
    print(num,total)
    rate = float(num)/total
    ratenum = int(50*rate)
    r = "[%s%s]" % ('>'*ratenum, ' '*(50-ratenum))
    #r = f"\\r[{'>'*ratenum}{' '*(30-ratenum)} ]({num,}/{total}) "
    return r

def subprocess(number):
    # 子进程
    callback_res = "call back"
    print('这是第{0}个子进程'.format(number))
    pid = os.getpid()              # 得到当前进程号
    print('当前进程号：{0}，开始时间：{1}'.format(pid, datetime.now().isoformat()))
    time.sleep(2)                 # 当前进程休眠30秒
    print('当前进程号：{0}，结束时间：{1}'.format(pid, datetime.now().isoformat()))
    return callback_res

if __name__ == '__main__':
    # 主进程
    print('这是主进程，进程编号：{0}'.format(os.getpid()))
    t_start = datetime.now()
    max = 4
    pool = multiprocessing.Pool(2)
    for i in range(4):
        pool.apply_async(subprocess, args=(i,), callback=log)
    pool.close()
    pool.join()
    t_end = datetime.now()
    print('主进程用时：{0}毫秒'.format((t_end - t_start).microseconds))

使用ProcessPoolExecutor 进行记录

python 线程池 ThreadPoolExecutor

import time
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, FIRST_COMPLETED, as_completed

def action(second):
    print(second)
    time.sleep(second)
    return second

lists = [4, 5, 2, 3]
all_task = []
with ThreadPoolExecutor(max_workers=2) as pool:
    for second in lists:
        all_task.append(pool.submit(action, second))

    for future in as_completed(all_task):
        print(f"{future.result()} 返回")

    print("----complete-----")

使用队列 queue 的方式进行返回

#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time    : 2018/5/23 14:15
# @Author  : zhouyuyao
# @File    : demon1.py
import queue,time
from concurrent.futures import ThreadPoolExecutor
def run(url_queue):
    while not url_queue.empty():
        url_list = url_queue.get()
        i = url_list[0]
        num = url_list[1]
    print(i*30)
    print(f"is Ok {num}")
    time.sleep(3)

if __name__ == '__main__':
    url_queue = queue.Queue(maxsize=5)
    num = 0
    all_task = []
    with ThreadPoolExecutor(max_workers=8) as pool:
        for url in range(20):
            url_queue.put([url, num])
            num += 1
            all_task.append(pool.submit(run, url_queue))