并行下载任务

Posted 2021-03-02 benlam

目标

• 通过 Python 的多线程、多进程等手段并发执行下载音频文件

MP3文本信息

• 只拿出几行作为例子显示，实际还有更多信息

《圆桌派》反派：怎么演“坏”人？
https://cdn.vistopia.com.cn/1536303525652.mp3
《圆桌派》误读：你常遇到会错意吗？
https://res.wx.qq.com/voice/getvoice?mediaid=MzU4MjI2OTA0Ml8yMjQ3NDkyODEw
《圆桌派》求知：眼花缭乱知识控
https://cdn.vistopia.com.cn/1536423336216.mp3

并行任务 --- concurrent.futures.ThreadPoolExecutor

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2020/3/31 11:54
# @File    : 并行任务-下载音频.py
# @Author  : BenLam
# @Link    : https://www.cnblogs.com/BenLam/
# @Version : PyCharm


from concurrent.futures import ThreadPoolExecutor


def open_music_urls():
    music = {}
    with open("mp3.txt", "r", encoding="utf-8") as f:
        a = [_.strip() for _ in f.readlines()]
        for _ in range(0, len(a), 2):
            music[a[_]] = a[_+1]
    return music

def down_mp3(name, urls):
	"""
	接受两个参数：
	name → 音频名字
	urls → 音频下载地址
	"""
    pwd = ".\\MP3\\"
    if not os.path.exists(pwd):
        os.mkdir(pwd)
    response = requests.get(urls)
    size = int(len(response.content))
    print(‘%r page is %d Kb | %s ‘ % (urls, (size//1024), name))

    with open(pwd + name + ".mp3", "ab")as f:
        f.write(response.content)
        f.flush()
    print("Missions Done ........")


if __name__ == ‘__main__‘:
    executor = ThreadPoolExecutor(max_workers=3)
    for name, url in open_music().items():
        future = executor.submit(down_mp3, name, url)
        print(future.done())
    print(‘主线程‘)

多线程 --- threading

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2020/3/31 11:54
# @File    : 多线程下载音频.py
# @Author  : BenLam
# @Link    : https://www.cnblogs.com/BenLam/
# @Version : PyCharm

import requests
import threading


def open_music_urls():
    music = {}
    with open("mp3.txt", "r", encoding="utf-8") as f:
        a = [_.strip() for _ in f.readlines()]
        for _ in range(0, len(a), 2):
            music[a[_]] = a[_+1]
    return music

def down_mp3(name, urls):
	"""
	接受两个参数：
	name → 音频名字
	urls → 音频下载地址
	"""
    pwd = ".\\MP3\\"
    if not os.path.exists(pwd):
        os.mkdir(pwd)
    response = requests.get(urls)
    size = int(len(response.content))
    print(‘%r page is %d Kb | %s ‘ % (urls, (size//1024), name))

    with open(pwd + name + ".mp3", "ab")as f:
        f.write(response.content)
        f.flush()
    print("Missions Done ........")


if __name__ == ‘__main__‘:
	Thread_list = []
	for name, urls in open_music_urls().items():
		t = threading.Thread(target=down_mp3, args=(name, urls))
		Thread_list.append(t)
	
	for _ in Thread_list:
		_.start()
	
	for _ in Thread_list:
		_.join()

多进程 --- multiprocessing

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2020/3/31 11:54
# @File    : 多进程下载音频.py
# @Author  : BenLam
# @Link    : https://www.cnblogs.com/BenLam/
# @Version : PyCharm

import requests
from multiprocessing import Process, Lock

def open_music_urls():
    music = {}
    with open("mp3.txt", "r", encoding="utf-8") as f:
        a = [_.strip() for _ in f.readlines()]
        for _ in range(0, len(a), 2):
            music[a[_]] = a[_+1]
    return music

def down_mp3(lock, name, urls):
	"""
	接受两个参数：
	name → 音频名字
	urls → 音频下载地址
	"""
    pwd = ".\\MP3\\"
    if not os.path.exists(pwd):
        os.mkdir(pwd)
	lock.acquire()
    try:
		response = requests.get(urls)
		size = int(len(response.content))
		print(‘%r page is %d Kb | %s ‘ % (urls, (size//1024), name))
		
		with open(pwd + name + ".mp3", "ab")as f:
			f.write(response.content)
			f.flush()
		print("Missions Done ........")
	finally:
        lock.release()

if __name__ == ‘__main__‘:
	lock = Lock()
	process_list = []
	for name, urls in open_music_urls().items():
		t = Process(target=down_mp3, args=(lock, name, urls))
		process_list.append(t)
	
	for _ in process_list:
		_.start()