r 対象クエリの网络ページへの语句出现频度を算出

Posted 2021-05-23

library(rvest)
library(plyr)
library(stringr)
library(RMeCab)


source_url <- "https://www.google.co.jp/search?q=BIツール"
# html <- read_html(source_url, encoding = "UTF-8")
html <- read_html(source_url, encoding = "Shift-JIS")

title_nodes <- html_nodes(html, "h3")
title_nodes <- title_nodes %>% html_nodes("a") %>% html_attr("href")

# TODO: ROOP IN TITLE_NODES
# url <- "http://www.google.co.jp/aclk?sa=l&ai=DChcSEwjbmP_Hy8HeAhWNvGQKHSAqDsIYABAAGgJwag&sig=AOD64_2Zb5Jqz5T-ZGayr2bkUggaSz25xA&ved=0ahUKEwjGsvvHy8HeAhXtHDQIHZQJABEQ0QwIEg&adurl="
for( url in title_nodes ){
  print(paste("Google",url,sep=":"))
  y <- try( html <- read_html(url, encoding = "UTF-8"),silent=FALSE )
  if ( class(y) == "try-error" )next

  list <- html_nodes(html, "a") %>% html_attr("href")
  # TODO: ROOP IN LIST
  # url <- "https://bi.lakeel.com/seminar/"
  for( url2 in list ){
    print(paste("link",url2,sep=":"))
    try( html <- read_html(url2, encoding = "UTF-8"), silent=FALSE )
    if( class(y) == "try-error" )next
  
    body <- html_nodes(html, "body") %>% html_text()
    body <- gsub("\n","",body)
    body <- gsub("\t","",body)
    body <- gsub("\r","",body)
    write(body,"body.txt")
    freq<-RMeCabFreq("body.txt")
    if( url2 == list[1] )freq.all<-freq
    else freq.all<-rbind(freq.all,freq)
    # TODO: freqを足し合わせ最終的に集計する
  }
  if( url == title_nodes[1] )text<-freq.all
  else text<-rbind(text,freq.all)
}

text2<-ddply(text,.(Term,Info1,Info2),summarize,Freqs=sum(Freq))
text2<-subset(text2,Info1=="名詞")