python 亚马逊欲しいものリストの取得·加工

Posted 2021-05-09

# 価格情報がある部分
soup.select('div#buybox .print-list-price')[0]
soup.select('div#buybox .kindle-price')[0]

# pandas
import pandas
buff2 = soup.select('div#buybox table')[0]
pandas.io.html.read_html(buff2.prettify())


soup = BeautifulSoup(buff,'html5lib')
tb = soup.select('div#buybox table')[0]
for tab in tb.find_all(True):
  del(tab['class'])

## 余分なアトリビュートを削除  
soup = BeautifulSoup(buff,'html5lib')
tb = soup.select('div#buybox table')[0]
for key in list(tb.a.attrs.keys()):
  del(tb.a[key])

### kindle-price で余分な空白改行を削除し、一行にまとめる
pattern = r'\s+'
rpattern = re.compile(pattern)

for j in soup.select('div#buybox .kindle-price'):
  for i in j.find_all(True):
    tmp = i.get_text().strip()
    tmp = re.sub(rpattern,' ',tmp)
    i.string = tmp
    
"".join(list(map(lambda x: x.get_text().strip(), soup.select('div#buybox .kindle-price td'))))
## 'Kindle 価格:￥ 562 ￥ 45の割引 (7%)'

# coding: utf-8
from bs4 import BeautifulSoup

f = open('amazon.html','r')
response = f.read()
soup = BeautifulSoup(response,'html5lib')
# 欲しいものリストからエントリを取り出すジェネレータ
wishlistID = "<< wishlistID >>"
def parse(soup,wishlistID):
    selecter = "div[data-id=@@@@@@@@@@@@]"
    selecter = selecter.replace('@@@@@@@@@@@@',wishlistID)
    for dataItem in soup.select(selecter):
        yield dataItem
    #END
# エントリから商品名を取り出す
def parse2(dataItem):
    target_id = "a#itemName_" + dataItem["data-itemid"]
    element = dataItem.select(target_id)[0]
    return element.text.strip()

for x in parse(soup,wishlistID):
    print(parse2(x))

driver.get('https://www.amazon.co.jp/gp/registry/wishlist/@@@@@@@@@@@@/')
tmp = driver.find_element_by_xpath('//a[contains(.,"See More")]')
nextURL = tmp.get_attribute('href')

# bs4
# soup.find_all("div", text=re.compile("See More"))[0].parent.parent['href']
# lxml.html
# dom.xpath('//a[contains(.,"See More")]')[0].attrib['href']

import lxml.html
f = open('./amazon2.html','r')
buf = f.read()
dom = lxml.html.fromstring(buf)
dom.xpath('//h5[contains(text(),"End of List")]')[0].text.strip()

from bs4 import BeautifulSoup
import re
soup = BeautifulSoup(buf, 'html5lib')
soup.find_all("h5", text=re.compile("End of List"))