缓存网页

Posted 2020-09-26 安迪_963

实现每个链接都能独立缓存，如果存在直接读取，如果不存在，则获取网页，然后系列化后保存到本地

目录功能比较简单：后续可能会改进

#!/usr/bin/env python
#coding:utf-8
#Created by Andy @ 2017/6/28


import os
import hashlib
import urllib.request
import random
import time
import gzip
import pickle

# 简单的反防爬，每次随机选下header
headers = [{‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 6.3 WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Maxthon/4.4.8.1000 Chrome/30.0.1599.101 Safari/537.36‘},
           {‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:45.0) Gecko/20100101 Firefox/45.0‘},
           {‘User-Agent‘: ‘Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 10.0; WOW64; Trident/7.0)‘}]


def cache_html(url, header):
	# 根据不同链接产生不同的哈希值，并以此为文件名，保证一链接一缓存文件
	md = hashlib.md5()
	md.update(url.encode(encoding=‘utf8‘))
	file_name = md.hexdigest()

	# 判断缓存文件是否已经存在，如果存在就直接读取缓存文件
	path = os.path.join(base_path, file_name)
	if os.path.exists(path) and os.path.getsize(path):
		print("Cache file already exist!")
		with open(path, ‘rb‘) as read_f:
			html = pickle.load(read_f)
			try:
				html = gzip.decompress(html).decode(‘utf8‘) # 有些网站的数据经过压缩了
			except:
				html = html.decode(‘utf8‘)
	else:
		req = urllib.request.Request(url, headers=header)
		html = urllib.request.urlopen(req).read()

		if not req or not html:
			print("Connection failed...")
		else:
			time.sleep(random.randint(1, 3))

			with open(file_name, ‘wb‘) as write_f:
				pickle.dump(html, write_f)

		try:
			html = gzip.decompress(html).decode(‘utf8‘)
		except:
			html = html.decode(‘utf-8‘)


	return html

if __name__ == ‘__main__‘:
	header = random.choice(headers)

	base_path = os.path.dirname(os.path.abspath(__file__))

	url = ‘http://www.python.org‘
	html = cache_html(url, header)
	print(html)