Web 使用rvest抓取网页上未显示的数据

Question

这是上一个问题的后续： Scraping data using R and puts results in a data frame

我正在尝试从 Glassdoor 上收集评论，包括子评分（工作与生活的平衡、文化和价值观等）。 子评级位于下拉菜单中，并显示为星数 (1-5)。 Dave2e 为我之前的问题发布了一个非常有用的解决方案，但我发现一些公司的评论页面格式不同，因此该解决方案不起作用。 下面是一个不工作的公司的例子。

library(stringr)
library(httr)  
library(xml2)  
library(rvest) 
library(purrr) 
library(tidyverse)
library(lubridate)

Subratings <- data.frame()
url <- "https://www.glassdoor.com/Reviews/Fresenius-Medical-Care-North-America-Reviews-"
settings_url <- ".htm?filter.iso3Language=eng"

for (x in 1:3) {
     pg_reviews <- read_html(GET(paste(url, "E10445", "_P", x, settings_url, sep = "")))
     
     #the ratings are stored in a data structure in a script
     #find all the scripts and then search
     scripts<-pg_reviews %>% html_elements(xpath='//script')
     
     #search the scripts for the ratings
     ratingsScript <- which(grepl("ratingCareerOpportunities", scripts))
     #filter the script down to just the data.  This is JSON like haven't figured out the beginning or end
     data1 <-scripts[ratingsScript] %>% html_text2() %>% str_extract("\"urlParams\":.+\\}\\}\\}\\}") 
     
     
     #extract the ratings
     WorkLifeBalance  <- str_extract_all(data1, '(?<="ratingWorkLifeBalance":)\\d') %>% unlist() %>% as.integer()
     CultureAndValues <- str_extract_all(data1, '(?<="ratingCultureAndValues":)\\d') %>% unlist() %>% as.integer()
     DiversityAndInclusion        <- str_extract_all(data1, '(?<="ratingDiversityAndInclusion":)\\d') %>% unlist() %>% as.integer()
     SeniorLeadership <- str_extract_all(data1, '(?<="ratingSeniorLeadership":)\\d') %>% unlist() %>% as.integer()
     CareerOpportunities <- str_extract_all(data1, '(?<="ratingCareerOpportunities":)\\d') %>% unlist() %>% as.integer()
     CompensationAndBenefits<- str_extract_all(data1, '(?<="ratingCompensationAndBenefits":)\\d') %>% unlist() %>% as.integer()
     
     #Combine columns
     combine <- cbind(WorkLifeBalance,CultureAndValues,DiversityAndInclusion,SeniorLeadership,
                      CareerOpportunities,CompensationAndBenefits)
     
     Subratings <- rbind(Subratings,combine)     
}

Answer 1

看起来这个页面少了一个右括号，试试： str_extract("\"urlParams\":.+\\}\\}\\}") 。
这也应该适用于前几页。

经过多次搜索，员工评论存储在以“reviews”开头并以}]}结尾的字符串中。
通过添加前导 { 将评论转换为有效的 JSON 并因此进行简单的转换。

library(stringr) 
library(httr)
library(xml2)
library(rvest) 
library(dplyr)

Subratings <- data.frame() 
url <- "https://www.glassdoor.com/Reviews/Fresenius-Medical-Care-North-America-Reviews-" 
settings_url <- ".htm?filter.iso3Language=eng"

dfs <- lapply(1:3, function(x) { 
   pg_reviews <- read_html(GET(paste(url, "E10445", "_P", x, settings_url, sep = "")))

   #the ratings are stored in a data structure in a script
   #find all the scripts and then search
   scripts<-pg_reviews %>% html_elements(xpath='//script')

   #search the scripts for the ratings
   ratingsScript <- which(grepl("ratingCareerOpportunities", scripts))

   #Extract text for the reviews from the script.  This is almost valid JSON format
   reviews <-scripts[ratingsScript] %>% html_text2() %>% 
                          str_extract("\"reviews\":.+?\\}\\]\\}") 
  # char <- nchar(reviews)  #debugging status

   #add a leading { to make valid JSON and convert
   answer <-jsonlite::fromJSON(paste("{", reviews))
   answer
})

bind_rows(dfs)