/
paragraphs_scrap.R
124 lines (84 loc) · 3.03 KB
/
paragraphs_scrap.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# scraping paragraphs from a website
#' Website text paragraph scraping
#'
#' @description This function is used to scrape text paragraphs from a website.
#'
#' @param link the link of the web page to scrape
#' @param contain filter the paragraphs according to the character string provided.
#' @param case_sensitive logical. Should the contain argument be case sensitive ? defaults to FALSE
#' @param collapse if TRUE the paragraphs will be collapsed into one element and the contain argument ignored.
#' @param askRobot logical. Should the function ask the robots.txt if we're allowed or not to scrap the web page ? Default is FALSE.
#' @return a character vector.
#' @examples \donttest{
#' # Extracting the paragraphs displayed on the health page of the New York Times
#'
#' link <- "https://www.nytimes.com/section/health"
#'
#' paragraphs_scrap(link)}
#'
#' @export
#' @importFrom rvest html_nodes html_text html_attr %>%
#' @importFrom xml2 read_html
#' @importFrom crayon green
#' @importFrom crayon bgRed
#' @importFrom robotstxt paths_allowed
#' @importFrom curl has_internet
paragraphs_scrap <- function(link,
contain = NULL,
case_sensitive = FALSE,
collapse = FALSE,
askRobot = FALSE) {
if (missing(link)) {
stop("'link' is a mandatory parameter")
}
if (!is.character(contain) && !is.null(contain)) {
stop("the 'contain' parameter must be passed a character string")
}
##################### Ask Robot part ###############################################
if (askRobot) {
if (paths_allowed(link) == TRUE) {
message(green("the robot.txt doesn't prohibit scraping this web page"))
} else {
message(bgRed(
"WARNING: the robot.txt doesn't allow scraping this web page"
))
}
}
####################################################################################
tryCatch(
expr = {
data <- link %>%
read_html() %>%
html_nodes("p") %>%
html_text()
if (is.null(contain) & collapse == FALSE) {
return(data)
} else if (is.null(contain) & collapse == TRUE) {
return(paste(data, collapse = " "))
} else if (!is.null(contain) & collapse == FALSE) {
return(data[grepl(contain,
data,
ignore.case = !case_sensitive)])
} else {
message("if you decide to collapse the paragraphs,
the contain argument will be ignored")
return(paste(data, collapse = " "))
}
},
error = function(cond) {
if (!has_internet()) {
message("Please check your internet connexion: ")
message(cond)
return(NA)
} else if (grepl("current working directory", cond) ||
grepl("HTTP error 404", cond)) {
message(paste0("The URL doesn't seem to be a valid one: ", link))
message(paste0("Here the original error message: ", cond))
return(NA)
} else {
message(paste0("Undefined Error: ", cond))
return(NA)
}
}
)
}