data_wrangling.Rmd

---
title: "Data Wrangling For FSL Invar experiments"
output:
  pdf_document:
    latex_engine: xelatex
---

```{r}
library(tidyverse)
library(brms)
library(lme4)
library(lmerTest)
library(plotrix)
library(stringr)
library(readxl)
library(RColorBrewer)
library(Hmisc)
```


# Verbal Argument Structure

Read in the data. Verb argument structure status (transitive, ambitransitive, intransitive) comes from CELEX2, distributed by the LDC.

```{r}
d = read.csv("test_items_downsample/argstruct-downsample_items.csv", header=FALSE) %>%
  rename("word" = V1) %>%
  rename("verb" = V2) %>%
  rename("pos" = V3) %>%
  rename("freq" = V4) %>%
  rename("is_trans" = V5) %>%
  rename("is_intrans" = V6) %>%
  rename("gram" = V7) %>%
  rename("target_id" = V8) %>%
  rename("item_number" = V9) %>%
  rename("test" = V10)

lstm_results = read.csv("test_items_downsample/argstruct-downsample_lstm_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 2) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "lstm")

ngram_results = read.csv("test_items_downsample/argstruct-downsample_ngram_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == ".", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  mutate(sent_pos = lag(sent_pos)) %>%
  mutate(model = "5gram")

rnng_results = read.csv("test_items_downsample/argstruct-downsample_rnng_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 2) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "rnng")

action_results = read.csv("test_items_downsample/argstruct-downsample_action_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 2) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "actionLstm")

v_counts = read_csv("data/v_counts.csv") %>%
  rename("freq" = `0`) %>%
  select(-X1, -LEMMA) %>%
  spread(XPOS, freq) %>%
  replace(is.na(.), 0) %>%
  mutate(total = VBD + VBN,
         percent_VBD = VBD/total,
         percent_VBN = VBN/total) %>%
  rename("word" = WORD)

d_filter = read_csv("data/v_counts.csv") %>%
  filter(XPOS != "VB") %>%
  select(-`0`, - X1) %>%
  group_by(LEMMA, XPOS) %>%
    mutate(posi = row_number()) %>%
  ungroup() %>%
  spread(XPOS, WORD) %>%
  drop_na() %>%
  filter(VBN != VBD)

d_args = read_csv("data/wsj_proportions.csv") %>%
  slice(0:14139) %>%
  select(-X1, -lemma) %>%
  mutate(has_dobj = if_else(has_dobj==TRUE, "obj", "no-obj")) %>%
  mutate(has_nsubjpass = if_else(has_nsubjpass == TRUE, "passubj", "no-passsubj")) %>%
  filter(has_nsubjpass == "no-passsubj") %>%
  select(-has_nsubjpass) %>%
  spread(has_dobj, count) %>%
  replace(is.na(.), 0) %>%
  mutate(p_obj = obj / (`no-obj` + obj)) %>%
  select(-`no-obj`, -obj) %>%
  spread(ptb_pos, p_obj) %>%
  select(-VBG, -VBP, -VBZ) %>%
  rename(
    VB_obj = VB,
    VBD_obj = VBD,
    VBN_obj = VBN
  )
  
```


Add in factor values based on token frequency:

  • "total_freq" counts the number of times the token shows up in the PTB
  
  • "vbd_freq" counts the number of times it appears in transitive contexts
  
  • "vbn_freq" counts the number of times it appears in passive contexts

```{r}
d_ngram = merge(d, ngram_results, by=0, all=TRUE)
d_lstm = merge(d, lstm_results, by=0, all=TRUE)
d_rnng = merge(d, rnng_results, by=0, all=TRUE)
d_action = merge(d, action_results, by=0, all=TRUE)
d_agg = rbind(d_ngram, d_lstm, d_rnng, d_action)


d_agg = d_agg %>%
  arrange(as.numeric(Row.names)) %>%
  filter((target_id-1 == sent_pos) | (target_id-2 == sent_pos) | (target_id-3 == sent_pos)) %>%
  filter( !((target_id-3 == sent_pos) & (test=="base-nomod")) ) %>%
  filter( !((target_id-3 == sent_pos) & (test=="base-pres")) ) %>%
  group_by(model, verb, pos, is_trans, is_intrans, gram, test, freq, item_number) %>%
    summarise(surprisal = sum(surprisal)) %>%
  ungroup() %>%
  spread(gram, surprisal) %>%
  mutate(obj_exp = nobj-obj) %>%
  select(-obj, -nobj) %>%
  mutate(is_trans = if_else(((is_trans=="Y") & (is_intrans=="Y")), "Ambitrans", as.character(is_trans))) %>%
  mutate(is_trans = if_else(is_trans == "Y", "Trans", is_trans)) %>%
  mutate(is_trans = if_else(is_trans == "N", "Intrans", is_trans)) %>%
  mutate(model = factor(model, levels = c("5gram", "lstm", "actionLstm", "rnng"))) %>%
  select(-is_intrans, -freq, -pos) %>%
  
  filter(! verb %in% d_filter$VBD) %>%
  filter(! verb %in% d_filter$VBN)

  
d_agg =  merge(d_agg, v_counts, by.x="verb", by.y = "word") %>%
  mutate(total = if_else(total > 100, 100, total)) %>%
  mutate(VB = if_else(VB > 100, 100, VB)) %>%
  mutate(VBD = if_else(VBD > 100, 100, VBD)) %>%
  mutate(VBN = if_else(VBN > 100, 100, VBN)) %>%

  mutate(vb_freq = "100") %>%
  mutate(vb_freq = if_else(VB<=50, "50", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=30, "30", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=20, "20", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=10, "10", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=5, "5", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=3, "3", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=2, "2", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=1, "1", vb_freq)) %>%
  mutate(vb_freq = if_else(VB<=0, "1", vb_freq)) %>%
  mutate(vb_freq = factor(vb_freq, levels = c("0", "1", "2", "3", "5", "10", "20", "30", "50", "100"))) %>%
  
  mutate(total_freq = "100") %>%
  mutate(total_freq = if_else(total<=50, "50", total_freq)) %>%
  mutate(total_freq = if_else(total<=30, "30", total_freq)) %>%
  mutate(total_freq = if_else(total<=20, "20", total_freq)) %>%
  mutate(total_freq = if_else(total<=10, "10", total_freq)) %>%
  mutate(total_freq = if_else(total<=5, "5", total_freq)) %>%
  mutate(total_freq = if_else(total<=3, "3", total_freq)) %>%
  mutate(total_freq = if_else(total<=2, "2", total_freq)) %>%
  mutate(total_freq = if_else(total<=1, "1", total_freq)) %>%
  mutate(total_freq = if_else(total<=0, "1", total_freq)) %>%
  mutate(total_freq = factor(total_freq, levels = c("0", "1", "2", "3", "5", "10", "20", "30", "50", "100"))) %>%
  
  mutate(vbd_freq = "100") %>%
  mutate(vbd_freq = if_else(VBD<=50, "50", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=30, "30", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=20, "20", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=10, "10", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=5, "5", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=3, "3", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=2, "2", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=1, "1", vbd_freq)) %>%
  mutate(vbd_freq = if_else(VBD<=0, "0", vbd_freq)) %>%
  mutate(vbd_freq = factor(vbd_freq, levels = c("0", "1", "2", "3", "5", "10", "20", "30", "50", "100"))) %>%
  
  mutate(vbn_freq = "100") %>%
  mutate(vbn_freq = if_else(VBN<=50, "50", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=30, "30", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=20, "20", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=10, "10", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=5, "5", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=3, "3", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=2, "2", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=1, "1", vbn_freq)) %>%
  mutate(vbn_freq = if_else(VBN<=0, "0", vbn_freq)) %>%
  mutate(vbn_freq = factor(vbn_freq, levels = c("0", "1", "2", "3", "5", "10", "20", "30", "50", "100")))

d_agg =  merge(d_agg, d_args, by.x="verb", by.y = "word") %>%
  filter((is_trans == "Intrans" & (VBD_obj < 0.1 | VB_obj < 0.1)) | (is_trans == "Trans" & (VBD_obj > 0.9 | VB_obj > 0.9 )))
  
write.csv(d_agg, "argstruct_data.csv")

```

# ====== NUMBER =======

## Read in Number-PP

These were generated seperatly

```{r}
d = read.csv("test_items_downsample/number-downsample_items.csv", header=FALSE) %>%
  rename("word" = V1) %>%
  rename("verb" = V2) %>%
  rename("pos" = V3) %>%
  rename("freq" = V4) %>%
  rename("gram" = V5) %>%
  rename("target_id" = V6) %>%
  rename("item_number" = V7) %>%
  rename("test" = V8) %>%
  filter(word != ".")

lstm_results = read.csv("test_items_downsample/number-downsample_lstm_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "lstm")

ngram_results = read.csv("test_items_downsample/number-downsample_ngram_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == ".", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != ".") %>%
  mutate(model = "5gram")

rnng_results = read.csv("test_items_downsample/number-downsample_rnng_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "rnng")

action_results = read.csv("test_items_downsample/number-downsample_action_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "actionLstm")

d_ngram = merge(d, ngram_results, by=0, all=TRUE)
d_lstm = merge(d, lstm_results, by=0, all=TRUE)
d_rnng = merge(d, rnng_results, by=0, all=TRUE)
d_action = merge(d, action_results, by=0, all=TRUE)
d_all = rbind(d_ngram, d_lstm, d_rnng, d_action)

```


## PP Modified Number

```{r}

d_pp = read.csv("test_items_downsample/number-pp_items.csv", header=FALSE) %>%
  rename("word" = V1) %>%
  rename("verb" = V2) %>%
  rename("pos" = V3) %>%
  rename("freq" = V4) %>%
  rename("gram" = V5) %>%
  rename("target_id" = V6) %>%
  rename("item_number" = V7) %>%
  rename("test" = V8) %>%
  filter(word != ".") %>%
  mutate(target_id = target_id - 1)

lstm_pp_results = read.csv("test_items_downsample/number-pp_lstm_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "lstm")

ngram_pp_results = read.csv("test_items_downsample/number-pp_ngram_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V3) %>%
  rename("surprisal" = V4) %>%
  select(-V1, -V2) %>%
  mutate(sent = if_else(word_1 == ".", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != ".") %>%
  mutate(model = "5gram")

rnng_pp_results = read.csv("test_items_downsample/number-pp_rnng_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "rnng")

action_pp_results = read.csv("test_items_downsample/number-pp_action_output.txt", sep="\t", header=FALSE) %>%
  rename("word_1" = V1) %>%
  rename("surprisal" = V2) %>%
  mutate(sent = if_else(word_1 == "<eos>", 1, 0)) %>%
  mutate(sent = cumsum(sent)) %>%
  mutate(sent_pos = 1) %>%
  group_by(sent) %>%
    mutate(sent_pos = cumsum(sent_pos)) %>%
  ungroup() %>%
  select(-sent) %>%
  mutate(sent_pos = sent_pos - 1) %>%
  filter(word_1 != "<eos>") %>%
  mutate(model = "actionLstm")

d_pp_ngram = merge(d_pp, ngram_pp_results, by=0, all=TRUE)
d_pp_lstm = merge(d_pp, lstm_pp_results, by=0, all=TRUE)
d_pp_rnng = merge(d_pp, rnng_pp_results, by=0, all=TRUE)
d_pp_action = merge(d_pp, action_pp_results, by=0, all=TRUE)
d_pp_all = rbind(d_pp_ngram, d_pp_lstm, d_pp_rnng, d_pp_action)

```

```{r}
d_all %>%
  arrange(as.numeric(Row.names)) %>%
  filter(target_id == sent_pos) %>%
  select(-Row.names, -word, -target_id, -word_1, -sent_pos) %>%
  filter(model == "5gram", test== "transf_simple") %>%
  filter(freq > 50) %>%
  spread(gram, surprisal) %>%
  mutate(pl_exp = sing-pl) %>%
  select(-sing, -pl) %>%
  mutate(acc = if_else(pl_exp > 0, 1, 0)) %>%
  mutate(acc = if_else(pos == "NN", 1-acc, acc)) %>%
  summarise(sum = n())

binom.test(408, 800, p=0.5, alternative = "greater")


```


```{r}

d_all = rbind(d_pp_all, d_all)

d_agg = d_all %>%
  arrange(as.numeric(Row.names)) %>%
  filter(target_id == sent_pos) %>%
  select(-Row.names, -word, -target_id, -word_1, -sent_pos) %>%
  spread(gram, surprisal) %>%
  mutate(pl_exp = sing-pl) %>%
  select(-sing, -pl) %>%
  mutate(model = factor(model, levels = c("5gram", "lstm", "actionLstm", "rnng"))) %>%
  mutate(test = factor(test, levels = c("base_simple", "base_pp", "base_rc", "transf_simple", "transf_mod"))) %>%
  mutate(freq = if_else(freq>100, 100, freq/1)) %>%
  mutate(freq_cat = "100") %>%
  #mutate(freq_cat = if_else(freq<=50, "50", freq_cat)) %>%
  #mutate(freq_cat = if_else(freq<=20, "20", freq_cat)) %>%
  mutate(freq_cat = if_else(freq<=10, "10", freq_cat)) %>%
  mutate(freq_cat = if_else(freq<=5, "5", freq_cat)) %>%
  mutate(freq_cat = if_else(freq<=4, "4", freq_cat)) %>%
  mutate(freq_cat = if_else(freq<=3, "3", freq_cat)) %>%
  mutate(freq_cat = if_else(freq<=2, "2", freq_cat)) %>%
  mutate(freq_cat = if_else(freq<=1, "1", freq_cat)) %>%
  mutate(freq_cat = factor(freq_cat, levels = c("1", "2", "3", "4", "5", "10", "100")))

#write.csv(d_agg, "number_data.csv")

```