python解析钩针图解到markdown文件/requests/BeautifulSoup

Posted 2021-06-08 Mars-xq

一、钩针图解之家

# coding=utf-8
import requests
from bs4 import BeautifulSoup

filename = '钩针图解之家/钩针图解之家.md'

if __name__ == '__main__':
    url = "https://mp.weixin.qq.com/s/ROqjMaJJEwHP7SNsGXs28w"
    homePage = requests.get(url)
    # print(res.encoding)  # ISO-8859-1
    homePage.encoding = "utf-8"
    # print(homePage.text)
    # print(type(res))  # <class 'requests.models.Response'>
    homePageSoup = BeautifulSoup(homePage.text, 'html.parser')
    homePageList = homePageSoup.select("section p a ")
    # print(homePageList)
    print("总共有 ： " + str(len(homePageList)))
    for index in range(len(homePageList)):
        # for li in homePageList:
        # print(li)
        href = homePageList[index]["href"]
        # print(href)

        detailPage = requests.get(href)
        detailPage.encoding = "utf-8"
        detailPageSoup = BeautifulSoup(detailPage.text, 'html.parser')
        detailPageList = detailPageSoup.select(".rich_pages")
        name = detailPageSoup.select("#activity-name")[0].text \\
            .replace('\\n', '').replace(' ', '').rstrip().lstrip()
        print(name)
        if index % 5 == 0:
            filename = '钩针图解之家/' + str(index) + '.md'
        with open(filename, 'a', encoding='utf-8') as file_object:
            file_object.write("# [" + str(index + 1) + "、" + name + "](" + href + ")")
            file_object.write("\\n")
            for value in detailPageList:
                try:
                    print(value)
                    src = value['data-src']
                    print(src)
                    if src.endswith('=gif'):
                        print('---->gif图')
                    elif src == 'https://mmbiz.qpic.cn/mmbiz_jpg/HicjZk3q41ZPiaBxDmK6njcibLYFutxsGjuSqqq37umEMqOZeyMXR2EiakAoM1CEL2UJcbuYq4IS7LricActVPlSeCw/640?wx_fmt=jpeg':
                        print('---->二维码图片')
                    elif src == 'https://mmbiz.qpic.cn/mmbiz_jpg/HicjZk3q41ZPiaBxDmK6njcibLYFutxsGjuyVpqBS1Oib65U4xJYuTsfibkibewuibFOUxu4POCz2ypQ8icTI7fjXTlEXA/640?wx_fmt=jpeg':
                        print('---->结束图片')
                    else:
                        file_object.write("![](" + src + ")")
                        file_object.write("\\n")
                except:
                    print('------error-------')
                    continue
                finally:
                    print('-------finally-----------')
            file_object.write("\\n")
            file_object.write("\\n")
            file_object.write("\\n")
            file_object.write("\\n")
        print('-------------------------------------->')
    print('===========================大结局===============================')

二、猫小姐的毛线屋

# coding=utf-8
import requests
from bs4 import BeautifulSoup

filename = '猫小姐的毛线屋/猫小姐的毛线屋111.md'

if __name__ == '__main__':
    url = "https://mp.weixin.qq.com/s/iyzHblpOiLG2x_hxRyj8eA"
    homePage = requests.get(url)
    # print(res.encoding)  # ISO-8859-1
    homePage.encoding = "utf-8"
    # print(res.text)
    # print(type(res))  # <class 'requests.models.Response'>
    homePageSoup = BeautifulSoup(homePage.text, 'html.parser')
    homePageList = homePageSoup.select("td p a")
    # for li in homePageList:
    print("总共有 ： " + str(len(homePageList)))
    for index in range(len(homePageList)):
        # for index in range(len(fruits)):
        #    print '当前水果 :', fruits[index]

        # print(li)
        href = homePageList[index]["href"]
        # print(href)

        detailPage = requests.get(href)
        detailPage.encoding = "utf-8"
        detailPageSoup = BeautifulSoup(detailPage.text, 'html.parser')
        detailPageList = detailPageSoup.select(".rich_pages")
        name = detailPageSoup.select("#activity-name")[0].text \\
            .replace('\\n', '').replace(' ', '').rstrip().lstrip()
        print(name)
        if index % 5 == 0:
            filename = '猫小姐的毛线屋/' + str(index) + '.md'
        with open(filename, 'a', encoding='utf-8') as file_object:
            file_object.write("# [" + str(index + 1) + "、" + name + "](" + href + ")")
            file_object.write("\\n")
            file_object.write("\\n")
            for value in detailPageList:
                src = value['data-src']
                print(src)
                if src.endswith('=gif'):
                    print('---->gif图')
                elif src == 'https://mmbiz.qpic.cn/mmbiz_jpg/HicjZk3q41ZPiaBxDmK6njcibLYFutxsGjuSqqq37umEMqOZeyMXR2EiakAoM1CEL2UJcbuYq4IS7LricActVPlSeCw/640?wx_fmt=jpeg':
                    print('---->二维码图片')
                elif src == 'https://mmbiz.qpic.cn/mmbiz_jpg/HicjZk3q41ZPiaBxDmK6njcibLYFutxsGjuyVpqBS1Oib65U4xJYuTsfibkibewuibFOUxu4POCz2ypQ8icTI7fjXTlEXA/640?wx_fmt=jpeg':
                    print('---->结束图片')
                else:
                    file_object.write("![](" + src + ")")
                    file_object.write("\\n")
            file_object.write("\\n")
            file_object.write("\\n")
            file_object.write("\\n")
            file_object.write("\\n")
        print('-------------------------------------->')
    print('===========================大结局===============================')