进程相关

Posted 2021-01-05 shanae

一、进程

import multiprocessing
import threading

data_list = []

def task(arg):
    data_list.append(arg)
    print(data_list)


def run():
    for i in range(10):
        p = multiprocessing.Process(target=task,args=(i,))
        # p = threading.Thread(target=task,args=(i,))
        p.start()

if __name__ == ‘__main__‘:
    run()

1.进程间的数据不共享

2.进程常用功能

import multiprocessing
import threading

class MyProcess(multiprocessing.Process):

    def run(self):
        print(‘当前进程‘,multiprocessing.current_process())

def run():
    p1 = MyProcess()
    p1.start()

    p2 = MyProcess()
    p2.start()

if __name__ == ‘__main__‘:
    run()

3.通过继承方式创建进程

在正常情况下，进程是不能进行数据共享的，但是当一定要进行数据共享时，需要用到以下方法

import multiprocessing
import threading
import queue
import time

q = multiprocessing.Queue()

def task(arg,q):
    q.put(arg)


def run():
    for i in range(10):
        p = multiprocessing.Process(target=task, args=(i, q,))
        p.start()

    while True:
        v = q.get()
        print(v)
run()

1.程间的数据共享:multiprocessing.Queue

import multiprocessing
import threading
import queue
import time

def task(arg,dic):
    time.sleep(2)
    dic[arg] = 100

if __name__ == ‘__main__‘:
    m = multiprocessing.Manager()
    
    process_list = []
    for i in range(10):
        p = multiprocessing.Process(target=task, args=(i,dic,))
        p.start()

        process_list.append(p)

    while True:
        count = 0
        for p in process_list:
            if not p.is_alive():
                count += 1
        if count == len(process_list):
            break
    print(dic)

2.进程间的数据共享:Manager

 　　1、进程锁

import time
import threading
import multiprocessing

lock = multiprocessing.RLock()

def task(arg):
    print(‘鬼子来了‘)
    lock.acquire()
    time.sleep(2)
    print(arg)
    lock.release()

if __name__ == ‘__main__‘:
    p1 = multiprocessing.Process(target=task,args=(1,))
    p1.start()

    p2 = multiprocessing.Process(target=task, args=(2,))
    p2.start()

　　2、进程池

import time
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor

def task(arg):
    time.sleep(2)
    print(arg)

if __name__ == ‘__main__‘:

    pool = ProcessPoolExecutor(5)
    for i in range(10):
        pool.submit(task,i)

初识爬虫

import requests
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor


# 模拟浏览器发送请求
# 内部创建 sk = socket.socket()
# 和抽屉进行socket连接 sk.connect(...)
# sk.sendall(‘...‘)
# sk.recv(...)

def task(url):
    print(url)
    r1 = requests.get(
        url=url,
        headers={
            ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/69.0.3497.92 Safari/537.36‘
        }
    )

    # 查看下载下来的文本信息
    soup = BeautifulSoup(r1.text,‘html.parser‘)
    print(soup.text)
    # content_list = soup.find(‘div‘,attrs={‘id‘:‘content-list‘})
    # for item in content_list.find_all(‘div‘,attrs={‘class‘:‘item‘}):
    #     title = item.find(‘a‘).text.strip()
    #     target_url = item.find(‘a‘).get(‘href‘)
    #     print(title,target_url)

def run():
    pool = ThreadPoolExecutor(5)
    for i in range(1,50):
        pool.submit(task,‘https://dig.chouti.com/all/hot/recent/%s‘ %i)


if __name__ == ‘__main__‘:
    run()

View Code