twitter_codecentric.Rmd

---
title: Combining Social Network Analysis and Topic Modeling to characterize codecentric's
  Twitter friends and followers
author: "Shirin Glander"
date: "July 13 2017"
output:
  prettydoc::html_pretty:
    highlight: github
    theme: cayman
---

```{r }
library(twitteR)
library(tidyverse)
library(tidytext)
```

```{r }
consumerKey = "INSERT KEY HERE"
consumerSecret = "INSERT SECRET KEY HERE"
accessToken = "INSERT TOKEN HERE"
accessSecret = "INSERT SECRET TOKEN HERE"
```

```{r }
options(httr_oauth_cache = TRUE)

setup_twitter_oauth(consumer_key = consumerKey, 
                    consumer_secret = consumerSecret, 
                    access_token = accessToken, 
                    access_secret = accessSecret)
```

```{r }
user <- getUser("codecentric")

friends <- user$getFriends() # who I follow
friends_df <- twListToDF(friends) %>%
  rownames_to_column()
followers <- user$getFollowers() # my followers
followers_df <- twListToDF(followers) %>%
  rownames_to_column()
```

```{r }
for (i in 1:length(friends)) {
  friends2 <- friends[[i]]$getFriends() # my friends' friends
  friends2_df <- twListToDF(friends2) %>%
    rownames_to_column() %>%
    mutate(friend = as.character(friends[[i]]$id))
  
  if (i == 1) {
    friends2_df_final <- friends2_df
  } else {
    friends2_df_final <- rbind(friends2_df_final, friends2_df)
  }
  print(i)
}
```

```{r }
friends_followers_df <- rbind(mutate(followers_df, type = ifelse(screenName %in% friends_df$screenName, "friend & follower", "follower")),
      mutate(friends_df, type = ifelse(screenName %in% followers_df$screenName, "friend & follower", "friend"))) %>%
  unique()
#summary(as.factor(friends_followers_df$type))
```

```{r languages, fig.width=5, fig.height=5, fig.align="center"}
friends_followers_df %>%
  group_by(type) %>%
  count(lang) %>%
  top_n(10) %>%
  droplevels() %>%
  ggplot(aes(x = reorder(lang, desc(n)), y = n)) +
    facet_grid(type ~ ., scales = "free") +
    geom_bar(stat = "identity", color = "#377F97", fill = "#377F97", alpha = 0.8) +
    theme_bw() +
    theme(strip.background=element_rect(fill = "#4A9888")) +
    theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust = 1)) +
    labs(x = "language ISO 639-1 code",
         y = "number of friends / followers",
         title = "Top 10 languages of friends and followers",
         caption = expression("Twitter friends and followers of @codecentric (data from July"~15^th~"2017)"))
```

```{r }
top_fol <- followers_df %>%
  mutate(date = as.Date(created, format = "%Y-%m-%d"),
         today = as.Date("2017-06-07", format = "%Y-%m-%d"),
         days = as.numeric(today - date),
         statusesCount_pDay = statusesCount / days) %>%
  select(screenName, followersCount, statusesCount_pDay) %>%
  arrange(desc(followersCount)) %>%
  .[1:10, ]
```

```{r }
top_tweet <- followers_df %>%
  mutate(date = as.Date(created, format = "%Y-%m-%d"),
         today = as.Date("2017-06-07", format = "%Y-%m-%d"),
         days = as.numeric(today - date),
         statusesCount_pDay = statusesCount / days) %>%
  select(screenName, followersCount, statusesCount_pDay) %>%
  arrange(desc(statusesCount_pDay)) %>%
  .[1:10, ]

top_fol_tweet <- rbind(top_fol, top_tweet) %>%
  unique()
```

```{r followers, fig.width=8, fig.height=4, fig.align="center"}
followers_df %>%
  mutate(date = as.Date(created, format = "%Y-%m-%d"),
         today = as.Date("2017-06-07", format = "%Y-%m-%d"),
         days = as.numeric(today - date),
         statusesCount_pDay = statusesCount / days) %>%
  ggplot(aes(x = followersCount, y = statusesCount_pDay)) +
    geom_smooth(method = "lm", color = "#377F97") +
    geom_point(color = "#4A9888", alpha = 0.6) +
    #geom_text(data = top_fol_tweet, aes(label = screenName), check_overlap = TRUE, size = 2) +
    scale_x_continuous(trans='log2') +
    scale_y_continuous(trans='log2') +
    theme_bw() +
    labs(x = expression(log[2]~"number of followers"),
         y = expression(log[2]~"average nr. of tweets per day"),
         title = "Codecentric's most influential followers",
         #subtitle = "Text labels show the top 10 followers with most tweets per day and highest number of followers (screen names)",
         caption = expression(2^nd~"degree followers and tweet rate of @codecentric Twitter followers (data from July"~15^th~"2017)"))
```

```{r }
top_fol2 <- followers_df %>%
  mutate(date = as.Date(created, format = "%Y-%m-%d"),
         today = as.Date("2017-06-07", format = "%Y-%m-%d"),
         days = as.numeric(today - date),
         statusesCount_pDay = statusesCount / days) %>%
  select(screenName, followersCount, statusesCount_pDay) %>%
  mutate(score = followersCount * statusesCount_pDay) %>%
  arrange(desc(score)) %>%
  .[1:100, ]
```

```{r }
top_fol_tweet2 <- top_fol2 %>%
  left_join(select(followers_df, screenName, description), by = "screenName") %>%
  mutate(id = seq_along(1:n()))
```

```{r }
library(tidytext)
library(SnowballC)
```

```{r }
data(stop_words)
stop_words_deu <- read.table("../german_stop_words.txt", header = FALSE)

tidy_descr <- top_fol_tweet2 %>%
  unnest_tokens(word, description) %>%
  mutate(word_stem = wordStem(word)) %>%
  anti_join(stop_words, by = "word") %>%
  anti_join(stop_words_deu, by = c("word" = "V1")) %>%
  filter(!grepl("\\.|http", word))
```

```{r }
library(wordcloud)
library(tm)
```

```{r wordcloud, fig.width=6, fig.height=6, fig.align="center"}
tidy_descr %>%
  count(word_stem) %>%
  mutate(word_stem = removeNumbers(word_stem)) %>%
  with(wordcloud(word_stem, n, max.words = 100, colors = c("#377F97", "#5BB18E")))
```

```{r }
tidy_descr_ngrams <- top_fol_tweet2 %>%
  unnest_tokens(bigram, description, token = "ngrams", n = 2) %>%
  filter(!grepl("\\.|http", bigram)) %>%
  separate(bigram, c("word1", "word2"), sep = " ") %>%
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word)

bigram_counts <- tidy_descr_ngrams %>%
  count(word1, word2, sort = TRUE)
```

```{r }
library(igraph)
library(ggraph)
```

```{r }
bigram_graph <- bigram_counts %>%
  filter(n > 1) %>%
  graph_from_data_frame()

set.seed(1)

a <- grid::arrow(type = "closed", length = unit(.15, "inches"))
```

```{r bigrams, fig.width=8, fig.height=5, fig.align="center"}
ggraph(bigram_graph, layout = "nicely") +
  geom_edge_link(aes(edge_alpha = n), show.legend = FALSE,
                 arrow = a, end_cap = circle(.07, 'inches')) +
  geom_node_point(color =  "#377F97", size = 5, alpha = 0.8) +
  geom_node_text(aes(label = name), vjust = 1.5, hjust = 0.5) +
  theme_void()
```

```{r }
bigrams_separated <- top_fol_tweet2 %>%
  unnest_tokens(bigram, description, token = "ngrams", n = 2) %>%
  filter(!grepl("\\.|http", bigram)) %>%
  separate(bigram, c("word1", "word2"), sep = " ") %>%
  filter(word1 == "not" | word1 == "no" | word1 == "nicht" | word1 == "kein") %>%
  filter(!word2 %in% stop_words$word)
```

```{r }
tidy_descr_sentiment <- tidy_descr %>%
  left_join(select(bigrams_separated, word1, word2), by = c("word" = "word2")) %>%
  inner_join(get_sentiments("nrc"), by = "word") %>%
  inner_join(get_sentiments("bing"), by = "word") %>%
  rename(nrc = sentiment.x, bing = sentiment.y) %>%
  mutate(nrc = ifelse(!is.na(word1), NA, nrc),
         bing = ifelse(!is.na(word1) & bing == "positive", "negative", 
                       ifelse(!is.na(word1) & bing == "negative", "positive", bing)))
```

```{r sentiment, fig.width=8, fig.height=3, fig.align="center"}
tidy_descr_sentiment %>%
  count(screenName, word, bing) %>%
  group_by(screenName, bing) %>%
  summarise(sum = sum(n)) %>%
  spread(bing, sum, fill = 0) %>%
  mutate(sentiment = positive - negative) %>%
  ggplot(aes(x = sentiment)) +
    geom_density(color = "#377F97", fill = "#377F97", alpha = 0.8) +
    theme_bw() +
    labs(title = "Sentiment analysis of codecentric's most influential followers",
         caption = expression("Top 100 most influential @codecentric Twitter followers on July"~15^th~2017))
```

```{r }
top_fol_tweet2 %>%
  left_join(select(followers_df, screenName, lang), by = "screenName") %>%
  count(lang)
```

```{r }
library(ggraph)
library(igraph)
```

```{r }
friends2_df_final <- friends2_df_final %>%
  left_join(select(friends_df, screenName, rowname), by = c("friend" = "rowname"))
```

```{r }
edge_table_1 <- data.frame(source = rep("codecentric", nrow(friends_df)),
                         target = friends_df$screenName)

edge_table_2 <- data.frame(source = followers_df$screenName,
                         target = rep("codecentric", nrow(followers_df)))

edge_table_3 <- data.frame(source = friends2_df_final$screenName.y,
                         target = friends2_df_final$screenName.x)

edge_table <- rbind(edge_table_1, edge_table_2, edge_table_3)
```

```{r }
graph <- graph_from_data_frame(edge_table, directed = TRUE)
layout <- layout_with_fr(graph)
V(graph)$color <- ifelse(V(graph)$name == "codecentric", "#377F97", "#4A9888")
V(graph)$size <- ifelse(V(graph)$name == "codecentric", 6, 1)
V(graph)$label <- ifelse(V(graph)$name == "codecentric", "codecentric", NA)
```

```{r }
pdf("twitter_net.pdf", width = 70, height = 80)
plot(graph,
     layout = layout,
     vertex.label = V(graph)$label,
     vertex.color = scales::alpha(V(graph)$color, alpha = 0.5), 
     vertex.size = V(graph)$size , 
     vertex.frame.color = "gray", 
     vertex.label.color = "black", 
     vertex.label.cex = 10,
     edge.arrow.size = 1)
dev.off()
```

```{r}
betweenness <- igraph::betweenness(graph, directed = TRUE)
#betweenness[order(betweenness, decreasing = TRUE)]
edge_betweenness <- igraph::edge_betweenness(graph, directed = TRUE)

V(graph)$size <- ifelse(V(graph)$name == "codecentric", 10, betweenness * 0.000001)
```

```{r }
pdf("twitter_net_betweenness.pdf", width = 70, height = 80)
plot(graph,
     layout = layout,
     vertex.label = V(graph)$label,
     vertex.color = scales::alpha(V(graph)$color, alpha = 0.5), 
     vertex.size = V(graph)$size, 
     vertex.frame.color = "gray", 
     vertex.label.color = "black", 
     vertex.label.cex = 6,
     edge.width = edge_betweenness * 0.0000001,
     edge.arrow.size = 1)
dev.off()
```

```{r }
library(topicmodels)
```

```{r }
friends2_df_final <- rename(friends2_df_final, screenName = screenName.x)
all_descr <- rbind(select(friends_df, screenName, description),
                   select(followers_df, screenName, description),
                   select(friends2_df_final, screenName, description)) %>%
  unnest_tokens(word, description) %>%
  mutate(word_stem = wordStem(word)) %>%
  anti_join(stop_words, by = "word") %>%
  anti_join(stop_words_deu, by = c("word" = "V1")) %>%
  filter(!grepl("\\.|http", word))
```

```{r }
dtm_words_count <- all_descr %>%
  mutate(word_stem = removeNumbers(word_stem)) %>%
  count(screenName, word_stem, sort = TRUE) %>%
  ungroup() %>%
  filter(word_stem != "") %>%
  cast_dtm(screenName, word_stem, n)

# set a seed so that the output of the model is predictable
dtm_lda <- LDA(dtm_words_count, k = 5, control = list(seed = 1234))

topics_beta <- tidy(dtm_lda, matrix = "beta")
```

```{r comparecloud, fig.width=10, fig.height=10, fig.align="center"}
library(reshape2)
library(RColorBrewer)
topics_beta %>%
  group_by(term) %>%
  top_n(1, beta) %>%
  group_by(topic) %>%
  top_n(50, beta) %>%
  acast(term ~ topic, value.var = "beta", fill = 0) %>%
  comparison.cloud(colors = brewer.pal(5, "Set1"))
```

```{r }
topics_gamma <- tidy(dtm_lda, matrix = "gamma") %>%
  arrange(desc(gamma))
```

```{r }
user_topic <- topics_gamma %>%
  group_by(document) %>%
  top_n(1, gamma)

node_table <- data.frame(name = unique(c(as.character(edge_table$source), as.character(edge_table$target)))) %>%
  left_join(user_topic, by = c("name" = "document")) %>%
  unique()

node_table <- node_table[!duplicated(node_table$name), ]

library(RColorBrewer)
pal <- brewer.pal(5, "Set1")

node_table$color = ifelse(node_table$topic == 1, pal[1],
                          ifelse(node_table$topic == 2, pal[2],
                                 ifelse(node_table$topic == 3, pal[3],
                                        ifelse(node_table$topic == 4, pal[4], pal[5]))))
```

```{r }
graph2 <- graph_from_data_frame(edge_table, directed = TRUE, vertices = node_table)
V(graph2)$size <- ifelse(V(graph2)$name == "codecentric", 4, 1)
V(graph2)$label <- ifelse(V(graph2)$name == "codecentric", "codecentric", NA)
```

```{r }
pdf("twitter_net_topics.pdf", width = 70, height = 80)
plot(graph2,
     layout = layout,
     vertex.label = V(graph)$label,
     vertex.color = scales::alpha(V(graph2)$color, alpha = 0.4), 
     vertex.size = V(graph2)$size , 
     vertex.frame.color = scales::alpha(V(graph2)$color, alpha = 0.4), 
     vertex.label.color = scales::alpha("black", alpha = 1), 
     vertex.label.cex = 8,
     edge.color = scales::alpha("grey", alpha = 0.4),
     edge.arrow.size = 1)
legend("topright", legend = c("Topic 1", "Topic 2", "Topic 3", "Topic 4", "Topic 5"), pch = 19,
       col = pal, pt.cex = 10, cex = 8, bty = "n", ncol = 1,
       title = "Node color") 
dev.off()
```