使用concurrent.futures和ProcessPoolExecutor来替代线程和进程

Posted 2020-10-06 pyrene

concurrent.futures和ProcessPoolExecutor这两个类实现的借口分别在不同的线程或进程中执行可调用的对象，这两个类在内部维护者一个工作线程或进程池，以及要执行的队列，这两个借口抽象的层级很高，无需关注实现细节

 普通方法实现下载国旗

import os
import time
import sys


import requests


POP20_CC=(‘CN IN US ID BR PK NG BD RU JP‘
          ‘MX PH VN ET EG DE IR TR CD FR‘).split()

BASE_URL=‘http://flupy.org/data/flags‘

DEST_DIR=‘downloads/‘

def save_flag(img,filename):
    path=os.path.join(DEST_DIR,filename)
    with open(path,‘wb‘)as fp:
        fp.write(img)

def get_flag(cc):
    url=‘{}/{cc}/{cc}.gif‘.format(BASE_URL,cc=cc.lower())
    resp=requests.get(url)
    return resp.content

def show(text):
    print(text,end="")
    sys.stdout.flush()

def download_many(cc_list):
    for cc in sorted(cc_list):
        image=get_flag(cc)
        show(cc)
        save_flag(image,cc.lower()+‘.gif‘)

    return len(cc_list)

def main(download_many):
    to=time.time()
    count=download_many(POP20_CC)
    elapsed=time.time()
    msg=‘\n{} flags downloaded in {:.2f}s‘
    print(msg.format(count,elapsed))

if __name__==‘__main__‘:
    main(download_many)

 替代多线程方法

#!/usr/bin/env python
# -*- coding:utf-8 -*-
from concurrent import futures
import sys
# sys.path.append(r"F:\regtest\asyncio！")
# from qw import save_flag,get_flag,show ,main
import os
import time
import sys


MAX_WORKERS=20



import requests


POP20_CC=(‘CN IN US ID BR PK NG BD RU JP‘
          ‘MX PH VN ET EG DE IR TR CD FR‘).split()

BASE_URL=‘http://flupy.org/data/flags‘

DEST_DIR=‘downloads/‘
def save_flag(img,filename):
    path=os.path.join(DEST_DIR,filename)
    with open(path,‘wb‘)as fp:
        fp.write(img)

def get_flag(cc):
    url=‘{}/{cc}/{cc}.gif‘.format(BASE_URL,cc=cc.lower())
    resp=requests.get(url)
    return resp.content

def show(text):
    print(text,end="")
    sys.stdout.flush()

def download_one(cc):
    image=get_flag(cc)
    show(cc)
    save_flag(image,cc.lower()+‘.gif‘)
    return cc

def download_many(cc_list):
    workers=min(MAX_WORKERS,len(cc_list))
    with futures.ThreadPoolExecutor(workers) as executor:
        res=executor.map(download_one,sorted(cc_list))

    return len(list(res))


def main(download_many):
    to=time.time()
    count=download_many(POP20_CC)
    elapsed=time.time()
    msg=‘\n{} flags downloaded in {:.2f}s‘
    print(msg.format(count,elapsed))

if __name__==‘__main__‘:
    main(download_many)

 有了这个神器就再也不用耗费精力去创建线程池或者进程池，以及队列来处理问题了。～

关于如何创建线程池进程池，在python基础篇～