Skip to content
Permalink
master
Switch branches/tags
Go to file
 
 
Cannot retrieve contributors at this time

Keep calm and comply with GDPR

GDPR Fines

h/t to Bob Rudis for sharing the data source, and to Roel Hogervorst for the guide to scraping this data. He provided the bulk of the scraping code, and I added bit of additional data cleaning. The data this week comes from Privacy Affairs.

I have also included all the raw text (gdpr_text.tsv) for the actual GDPR legal documents, in case someone was interested in parsing through them or using them along with the violations.

Per Wikipedia GDPR is:

The General Data Protection Regulation (EU) 2016/679 (GDPR) is a regulation in EU law on data protection and privacy in the European Union (EU) and the European Economic Area (EEA). It also addresses the transfer of personal data outside the EU and EEA areas. The GDPR aims primarily to give control to individuals over their personal data and to simplify the regulatory environment for international business by unifying the regulation within the EU.[1] Superseding the Data Protection Directive 95/46/EC, the regulation contains provisions and requirements related to the processing of personal data of individuals (formally called data subjects in the GDPR) who reside in the EEA, and applies to any enterprise—regardless of its location and the data subjects' citizenship or residence—that is processing the personal information of data subjects inside the EEA.

Get the data here

# Get the Data

gdpr_violations <- readr::read_tsv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-04-21/gdpr_violations.tsv')
gdpr_text <- readr::read_tsv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-04-21/gdpr_text.tsv')

# Or read in with tidytuesdayR package (https://github.com/thebioengineer/tidytuesdayR)
# PLEASE NOTE TO USE 2020 DATA YOU NEED TO USE the tidytuesdayR version after Jan 2020.

# Either ISO-8601 date or year/week works!

# Install via devtools::install_github("thebioengineer/tidytuesdayR")

tuesdata <- tidytuesdayR::tt_load('2020-04-21')
tuesdata <- tidytuesdayR::tt_load(2020, week = 17)


gdpr_violations <- tuesdata$gdpr_violations

Data Dictionary

gdpr_violations.tsv

variable class description
id integer Idetifier for fine/violation
picture character SVG image of violation country flag
name character Name of country where violation was enforced
price integer Fine price in Euros (€)
authority character Authority that enacted the violation
date character Date of violation
controller character Controller of data - the violator
article_violated character Specific GDPR Article violated (see the gdpr_text.tsv data for specifics)
type character Type of violation
source character Original source (URL) of fine data
summary character Summary of violation

gdpr_text.tsv

variable class description
chapter double GDPR Chapter Number
chapter_title character Chapter title
article double GDPR Article number
article_title character Article title
sub_article double Sub article number
gdpr_text character Raw text of article/subarticle
href character URL to the raw text itself

Cleaning Script

library(tidyverse)
library(rvest)

# Note the following code was adapted from
# https://blog.rmhogervorst.nl/blog/2020/04/08/scraping-gdpr-fines/

link <- "https://www.privacyaffairs.com/gdpr-fines/"
page <- read_html(link)


temp <- page %>% html_nodes("script") %>% 
  .[9] %>% 
  rvest::html_text() 

ends <- str_locate_all(temp, "\\]")
starts <- str_locate_all(temp, "\\[")

table1 <- temp %>% 
  stringr::str_sub(start = starts[[1]][1,2], end = ends[[1]][1,1]) %>% 
  str_remove_all("\\\n") %>% 
  str_remove_all("\\\r") %>%
  jsonlite::fromJSON() %>% 
  as_tibble() %>% 
  mutate(summary = str_remove_all(summary,"<p>|</p>|\n"))


table2 <- temp %>% 
  stringr::str_sub(start = starts[[1]][2,2], end = ends[[1]][2,1]) %>% 
  str_remove_all("\\\n") %>% 
  str_remove_all("\\\r") %>%  
  jsonlite::fromJSON() %>% 
  as_tibble() %>% 
  mutate(summary = str_remove_all(summary,"<p>|</p>|\n"))


all_df <- bind_rows(table1, table2) %>% 
  janitor::clean_names() %>%
  mutate(
    authority = str_remove(authority, "\t"),
    article_violated = str_remove(article_violated, '<a href="https://www.privacy-regulation.eu/en/32.htm">') %>% 
           str_remove('</a>'),
    article_violated = str_replace_all(article_violated, ", Art", "|Art"),
    type = str_remove(type, '<a href="https://www.privacy-regulation.eu/en/32.htm">') %>% 
      str_remove('</a>')
           )

# most frequent articles violated
all_df %>% 
  separate_rows(article_violated, sep = "\\|") %>% 
  count(article_violated, sort = T)

all_df %>% 
  write_tsv("2020/2020-04-21/gdpr_violations.tsv")


# Getting the actual article text -----------------------------------------

raw_article <- "https://gdpr-info.eu/" %>% 
  read_html()

# Get all the urls for specific articles/chapters
gdpr_href <- raw_article %>% 
  html_node(xpath = '//*[@id="tablepress-12"]') %>% 
  html_nodes("a") %>% 
  html_attr("href")

# pull the titles as well
gdpr_titles <- raw_article %>% 
  html_node(xpath = '//*[@id="tablepress-12"]') %>% 
  html_nodes("a") %>% 
  html_attr("data-title")

# pull the numbers of article/chapters
gdpr_numbers <- raw_article %>% 
  html_node(xpath = '//*[@id="tablepress-12"]') %>% 
  html_nodes("a") %>% 
  html_text()

# put it all into a df
gdpr_df <- tibble(
  article = gdpr_numbers,
  title = str_trim(gdpr_titles),
  href = gdpr_href
) 

# Tidy up the data, create chapters vs articles
clean_gdpr <- gdpr_df %>% 
  mutate(chapter = if_else(str_length(article) > 3, article, NA_character_),
         chapter_title = if_else(str_length(article) > 3, title, NA_character_)) %>% 
  fill(chapter, chapter_title) %>% 
  filter(!str_detect(article, "Chapter")) %>% 
  mutate(article = as.double(article)) %>% 
  filter(!is.na(article)) %>% 
  select(starts_with("chapter"), article, article_title = title, href)

clean_gdpr

# LONG running outcome
# Get all the raw html from each of the urls for each article
all_articles <- clean_gdpr %>% 
  mutate(raw_html = map(href, read_html))

# function to take raw html and turn it into text for that specific article
get_gdpr_text <- function(html_in){
  
  test_var <- html_in %>% 
    html_node(".entry-content") %>% 
    html_nodes("ol") %>% 
    html_text()
  
  if (length(test_var) == 0){
   text <- html_in %>%
     html_node(".entry-content > p") %>% 
     html_text() %>% 
     str_remove("^[:digit:]") 
  } else {
    text <- html_in %>% 
      html_node(".entry-content") %>% 
      html_nodes("ol") %>% 
      html_text() %>% 
      .[[1]] %>% 
      str_replace_all(";\n", "\t") %>% 
      str_replace_all(":\n", "\t") %>% 
      str_split("\n") %>% 
      .[[1]] %>% 
      .[. != ""] %>% 
      str_replace_all("\t", "\n") %>% 
      str_remove("^[:digit:]")
  }
  
  
  text
    
}

# Test
get_gdpr_text(read_html("http://gdpr-info.eu/art-2-gdpr/"))

# unnest the list column of text
clean_articles <- all_articles %>% 
  mutate(gdpr_text = map(raw_html, get_gdpr_text)) %>% 
  unnest_longer(gdpr_text)

# final dataframe
final_articles <- clean_articles %>% 
  group_by(article) %>% 
  mutate(sub_article = row_number()) %>% 
  relocate(sub_article, .after = "article_title") %>% 
  relocate(gdpr_text, .after = "sub_article") %>% 
  ungroup() %>% 
  mutate(chapter = str_extract(chapter, "[:digit:]+")) %>% 
  mutate_at(vars(chapter, article, sub_article), as.double) %>% 
  select(-raw_html)

final_articles %>% view()

write_tsv(final_articles, "2020/2020-04-21/gdpr_text.tsv")