线程池and进程池实战

Posted 2021-06-02 J哥。

线程池and进程池的学习

# 线程池  一次性开辟一些线程,我们用户给线程池提交任务.
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor    #线程池 , 进程池

def fn(name):
    for i in range(1,1000):
        print(name,i)


if __name__ == '__main__':
    #创建线程池
    with ThreadPoolExecutor(50) as t:
        for i in range(1,100):
            t.submit(fn, name = f"线程{i}" )
        #等待线程池的任务 执行完毕,才进一步进行(守护)
        print('123')

线程池and进程池实战：

import requests
from lxml import etree
import csv
from  concurrent.futures import ThreadPoolExecutor


#保存方式:

f = open("4_5新价.csv", mode="w", encoding='utf-8')
csvwriter = csv.writer(f)


def download_one_page(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit'
                      '/537.36 (Khtml, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
    }
    resp = requests.get(url, headers=headers)
    resp.encoding = "utf-8"
    # print(resp.text)
    html = etree.HTML(resp.text)
    # tbody = html.xpath('/html/body/div[2]/div[4]/div[1]/table/')[0]
    tbody = html.xpath('/html/body/div[2]/div[4]/div[1]/table')[0]

    # print(tbody)
    tr = tbody.xpath('./tr[position()>1]')
    for td in tr:
        tds = td.xpath('./td/text()')
        # 对数据进行简单的处理  写一个生成器
        # print(td)
        tds = (item.replace("\\\\", "").replace("/", "") for item in tds)
        csvwriter.writerow(tds)



if __name__ == '__main__':
    # for i in range(1,314940):  #效率极其低下
        # download_one_page(f"http://www.xinfadi.com.cn/marketanalysis/0/list/{i}.shtml")



    with ThreadPoolExecutor(50) as t :
        for i in range (1,200):

            #r任务转手
            t.submit(download_one_page ,f"http://www.xinfadi.com.cn/marketanalysis/0/list/{i}.shtml")
print('提取完毕!')