Python高性能编程

Posted 2020-10-12

 

1.串行

import time
import requests

url_lists = [
    ‘http://www.baidu.com‘,
    ‘http://fanyi.baidu.com‘,
    ‘http://map.baidu.com‘,
    ‘http://music.baidu.com/‘,
    ‘http://tieba.baidu.com‘,
    ‘http://v.baidu.com‘,
    ‘http://image.baidu.com‘,
    ‘http://zhidao.baidu.com‘,
    ‘http://news.baidu.com‘,
    ‘http://xueshu.baidu.com‘]

start_time = time.time()
for url in url_lists:
    response = requests.get(url)
    print(response.text)
print("Runtime: {}".format(time.time()-start_time))

# Runtime: 1.95

　　

2.多进程

import time
import requests
from multiprocessing import Process

url_lists = [
    ‘http://www.baidu.com‘,
    ‘http://fanyi.baidu.com‘,
    ‘http://map.baidu.com‘,
    ‘http://music.baidu.com/‘,
    ‘http://tieba.baidu.com‘,
    ‘http://v.baidu.com‘,
    ‘http://image.baidu.com‘,
    ‘http://zhidao.baidu.com‘,
    ‘http://news.baidu.com‘,
    ‘http://xueshu.baidu.com‘]


def task(url):
    response = requests.get(url)
    print(response.text)

if __name__ == ‘__main__‘:
    p_list = []
    start_time = time.time()
    for url in url_lists:
        p = Process(target=task, args=(url,))
        p_list.append(p)
        p.start()
    for p in p_list:
        p.join()

    print("Runtime: {}".format(time.time() - start_time))

# Runtime: 1.91

　

3.进程池（1）

import time
import requests
from concurrent.futures import ProcessPoolExecutor

"""
Py2里    没有线程池   但是有进程池
Py3里    有线程池     有进程池
"""

url_lists = [
    ‘http://www.baidu.com‘,
    ‘http://fanyi.baidu.com‘,
    ‘http://map.baidu.com‘,
    ‘http://music.baidu.com/‘,
    ‘http://tieba.baidu.com‘,
    ‘http://v.baidu.com‘,
    ‘http://image.baidu.com‘,
    ‘http://zhidao.baidu.com‘,
    ‘http://news.baidu.com‘,
    ‘http://xueshu.baidu.com‘]

def task(url):
    response = requests.get(url)
    print(response.content)

if __name__ == ‘__main__‘:
    start_time = time.time()
    pool = ProcessPoolExecutor(10)
    for url in url_lists:
        pool.submit(task,url)
    pool.shutdown(wait=True)
    print("Runtime: {}".format(time.time() - start_time))

# Runtime: 2.00

　　

3.进程池（2）

import time
import requests
from  multiprocessing import Pool


url_lists = [
    ‘http://www.baidu.com‘,
    ‘http://fanyi.baidu.com‘,
    ‘http://map.baidu.com‘,
    ‘http://music.baidu.com/‘,
    ‘http://tieba.baidu.com‘,
    ‘http://v.baidu.com‘,
    ‘http://image.baidu.com‘,
    ‘http://zhidao.baidu.com‘,
    ‘http://news.baidu.com‘,
    ‘http://xueshu.baidu.com‘]

def task(url):
    response = requests.get(url)
    return response.content
def callBackFunc(content):
    print(content)

if __name__ == ‘__main__‘:
    start_time = time.time()
    pool = Pool(10)
    for url in url_lists:
        pool.apply_async(func=task,args=(url,),callback=callBackFunc)
    pool.close()
    pool.join()
    print("Runtime: {}".format(time.time() - start_time))

# Runtime: 1.96

　　

 

4.多线程

import time
import requests
from threading import Thread

url_lists = [
    ‘http://www.baidu.com‘,
    ‘http://fanyi.baidu.com‘,
    ‘http://map.baidu.com‘,
    ‘http://music.baidu.com/‘,
    ‘http://tieba.baidu.com‘,
    ‘http://v.baidu.com‘,
    ‘http://image.baidu.com‘,
    ‘http://zhidao.baidu.com‘,
    ‘http://news.baidu.com‘,
    ‘http://xueshu.baidu.com‘]


def task(url):
    response = requests.get(url)
    print(response.text)

if __name__ == ‘__main__‘:
    t_list = []
    start_time = time.time()
    for url in url_lists:
        t = Thread(target=task, args=(url,))
        t_list.append(t)
        t.start()
    for t in t_list:
        t.join()

    print("Runtime: {}".format(time.time() - start_time))

# Runtime: 0.49

　　

5.线程池

import time
import requests
from concurrent.futures import ThreadPoolExecutor

"""
Py2里    没有线程池   但是有进程池
Py3里    有线程池     有进程池
"""

url_lists = [
    ‘http://www.baidu.com‘,
    ‘http://fanyi.baidu.com‘,
    ‘http://map.baidu.com‘,
    ‘http://music.baidu.com/‘,
    ‘http://tieba.baidu.com‘,
    ‘http://v.baidu.com‘,
    ‘http://image.baidu.com‘,
    ‘http://zhidao.baidu.com‘,
    ‘http://news.baidu.com‘,
    ‘http://xueshu.baidu.com‘]

def task(url):
    response = requests.get(url)
    print(response.content)

if __name__ == ‘__main__‘:
    start_time = time.time()
    pool = ThreadPoolExecutor(10)
    for url in url_lists:
        pool.submit(task,url)
    pool.shutdown(wait=True)
    print("Runtime: {}".format(time.time() - start_time))

# Runtime: 0.51