r 解析CapFriendly NHL Salary网站

Posted 2021-05-23

options(stringsAsFactors = FALSE)

## load the packages
library(tidyverse)
library(rvest)

## settings
BASE_URL = "https://www.capfriendly.com/browse/active/2015&p=%s"
TABLE_NAME = "brwt"
TABLE_XPATH = '//*[@id="brwt"]'
PAGES = 1:29  ## there are 29 pages via the browser

## dataset to store our results
salaries = data.frame()

## loop and parse the data
for (i in PAGES) { 
  ## build the URL and get the page
  URL = sprintf(BASE_URL, i)
  resp = URL %>% read_html() %>% html_nodes(xpath=TABLE_XPATH)
  stats = resp[[1]] %>% html_table()
  ## make all strings to avoid type issues and bind the data
  stats2 = mutate_all(stats, funs(as.character(.)))
  salaries = bind_rows(salaries, stats2)
  ## cleanup
  rm(URL, resp, stats, stats2)
  ## status
  cat("finished ", i, "\n")
} #endfor


## save out the dataset
write_csv(salaries, "~/Downloads/salaries1415.csv", na="")