experiment-results.Rmd

---
title: "Dyconits Experiment Results"
output:
  html_notebook
---

Set working directory to the desired experiment and load data. This is the location of the underlying data for all figures below.
```{r setup, include=FALSE, echo=FALSE}
library(tidyverse)
theme_set(theme_bw())
library(knitr)
library(gridExtra)
library(grid)
library(cowplot)
library(data.table)
library(zoo)
library(stargazer)
library(ggpubr)
library(ggsci)

opts_knit$set(root.dir = "~/scratch/experiments/")

grid_arrange_shared_legend <-
  function(...,
           nrow = length(list(...)),
           ncol = 1,
           position = c("bottom", "right")) {
    
    plots <- list(...)
    position <- match.arg(position)
    g <-
      ggplotGrob(plots[[1]] + theme(legend.position = position))$grobs
    legend <- g[[which(sapply(g, function(x)
      x$name) == "guide-box")]]
    lheight <- sum(legend$height)
    lwidth <- sum(legend$width)
    gl <- lapply(plots, function(x)
      x + theme(legend.position = "none"))
    gl <- c(gl, ncol = ncol, nrow = nrow)
    
    combined <- switch(
      position,
      "bottom" = arrangeGrob(
        do.call(arrangeGrob, gl),
        legend,
        ncol = 1,
        heights = unit.c(unit(1, "npc") - lheight, lheight)
      ),
      "right" = arrangeGrob(
        do.call(arrangeGrob, gl),
        legend,
        ncol = 2,
        widths = unit.c(unit(1, "npc") - lwidth, lwidth)
      )
    )
    
    grid.newpage()
    grid.draw(combined)
    
    # return gtable invisibly
    invisible(combined)
    
  }


theme_Publication <- function(base_size=14, base_family="helvetica") {
      library(grid)
      library(ggthemes)
      (theme_foundation(base_size=base_size, base_family=base_family)
       + theme(plot.title = element_text(face = "bold",
                                         size = rel(1.2), hjust = 0.5),
               text = element_text(),
               panel.background = element_rect(colour = NA),
               plot.background = element_rect(colour = NA),
               panel.border = element_rect(colour = NA),
               axis.title = element_text(face = "bold",size = rel(1)),
               axis.title.y = element_text(angle=90,vjust =2),
               axis.title.x = element_text(vjust = -0.2),
               axis.text = element_text(), 
               axis.line = element_line(colour="black"),
               axis.ticks = element_line(),
               panel.grid.major = element_line(colour="#f0f0f0"),
               panel.grid.minor = element_blank(),
               legend.key = element_rect(colour = NA),
               legend.position = "bottom",
               legend.direction = "horizontal",
               legend.key.size= unit(0.2, "cm"),
               legend.spacing = unit(0, "cm"),
               legend.title = element_text(face="italic"),
               plot.margin=unit(c(10,5,5,5),"mm"),
               strip.background=element_rect(colour="#f0f0f0",fill="#f0f0f0"),
               strip.text = element_text(face="bold")
          ))
      
}

scale_fill_Publication <- function(...){
      library(scales)
      discrete_scale("fill","Publication",manual_pal(values = c("#386cb0","#fdb462","#7fc97f","#ef3b2c","#662506","#a6cee3","#fb9a99","#984ea3","#ffff33")), ...)

}

scale_colour_Publication <- function(...){
      library(scales)
      discrete_scale("colour","Publication",manual_pal(values = c("#386cb0","#fdb462","#7fc97f","#ef3b2c","#662506","#a6cee3","#fb9a99","#984ea3","#ffff33")), ...)

}

saveplot <- function(filename, ...) {
  ggsave2(filename, ...)
  knitr::plot_crop(filename)
}
```

# Scalability Improvement

```{r}
df <- fread("scalability-experiment/results/opencraft-events.log", sep="\t")

toHumanName <- function(v) {
  if (v == "policy-is") {
    "IS"
  } else if (v == "policy-isn") {
    "ISN"
  } else if (v == "policy-aoi") {
    "AOI"
  } else if (v == "policy-zero") {
    "ZERO"
  } else {
    "UNKOWN"
  }
}

getTickPartName <- function(v) {
  if (v == "tick") {
    "total"
  } else if (v == "tick_network_rx") {
    "net_rx"
  } else if (v == "tick_network_tx") {
    "net_tx"
  } else if (v == "tick_worlds") {
    "simulate"
  } else {
    "other"
  }
}

df %>%
  filter(key %in% c("tick", "tick_network_rx", "tick_worlds", "tick_network_tx")) %>%
  mutate(key = map_chr(key, getTickPartName)) %>%
  mutate(key = fct_rev(key)) %>%
  filter(config %in% c("policy-isn", "policy-is", "policy-zero", "policy-aoi")) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  group_by(key, iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  mutate(config = factor(config, levels = c("ZERO", "AOI", "ISN", "IS"))) %>%
  mutate(rollmeanvalue = rollmean(value, 100, fill = NA)) %>%
  mutate(markers = ifelse((row_number() + 1000) %% 2000 == 0, rollmeanvalue, NA)) %>%
  ggplot(aes(x=timestamp, y=rollmeanvalue, color=key)) +
  geom_line() +
  geom_point(aes(y=markers, shape=key), size=2.5) +
  geom_hline(yintercept=50, linetype="dashed", color = "red", size=1) +
  annotate("rect", xmin=-Inf, ymin=50, xmax=Inf, ymax=Inf, fill="red", alpha = 0.2) +
  labs(x="time (s)", y="tick duration (ms)", color = "component", shape = "component") +
  theme_half_open() +
  panel_border() +
  background_grid() +
  theme(legend.position = "bottom") +
  facet_wrap(vars(config)) +
  theme(strip.background=element_rect(fill='white', color="black"))
saveplot("numplayers-analysis.pdf", width=6, height=4)

df %>%
  filter(key %in% c("tick")) %>% #, "tick_network_rx", "tick_worlds", "tick_network_tx")) %>%
  mutate(key = map_chr(key, getTickPartName)) %>%
  mutate(key = fct_rev(key)) %>%
  filter(config %in% c("policy-zero", "policy-is")) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  group_by(key, iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  mutate(rollmeanvalue = rollmean(value, 100, fill = NA)) %>%
  mutate(markers = ifelse((row_number() + 1000) %% 2000 == 0, rollmeanvalue, NA)) %>%
  ggplot(aes(x=timestamp, y=rollmeanvalue, color=fct_rev(config))) +
  geom_line() +
  geom_point(aes(y=markers, shape=fct_rev(config)), size=3) +
  geom_hline(yintercept=50, linetype="dashed", color = "red", size=1) +
  annotate("rect", xmin=-Inf, ymin=50, xmax=Inf, ymax=Inf, fill="red", alpha = 0.2) +
  annotate("text", x=5, y = 55, label = "Game overloaded", fontface = "bold", size = 5, hjust="left") +
  annotate("text", x=5, y = 45, label = "Game operates normally", fontface = "bold", size = 5, hjust="left") +
  annotate("text", x=200, y = 13, label = "Better", size = 5, hjust = "center") +
  annotate("segment", x=100, y = 9, xend=300, yend=9, arrow=arrow(length = unit(0.15, "inches"))) +
  annotate("text", x=5, y = 30, label = "Worse", size = 5, hjust = "left") +
  annotate("segment", x=0, y = 20, xend=0, yend=40, arrow=arrow(length = unit(0.15, "inches"))) +
  labs(x="time (s) / number of players", y="tick duration (ms)", color = "", shape = "") +
  theme_half_open() +
  theme(legend.justification=c(1,0), legend.position=c(1,0)) +
  scale_color_manual(values = c("#868686", "#CD534C"), labels = c("without Dyconits", "with Dyconits")) +
  scale_shape_discrete(labels = c("without Dyconits", "with Dyconits"))
saveplot("numplayers-motivation.pdf", width=6, height=2.4)

df %>%
  filter(key == "numplayers") %>%
  filter(config == "policy-isn" | config == "policy-is" | config == "policy-zero" | config == "policy-aoi") %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  group_by(iteration, config) %>%
  summarize(value=max(value)) %>%
  mutate(config = fct_relevel(config, "ZERO", "AOI", "ISN", "IS")) %>%
  ggplot(aes(x=fct_rev(config), y=value)) +
  geom_col(aes(fill=config), position = "dodge2", stat="identity", width=0.5) +
  scale_y_continuous(expand = expansion(mult = c(0, .1))) +
  labs(x="Dyconit policy", y="number of players") +
  # Uncomment lines below to add (hardcoded) annotations
  # annotate("segment", x=-Inf, xend=Inf, y=350, yend=350, linetype="dashed") +
  # annotate("text", y=400, x=2, label="+11%", size=5, hjust="left") +
  # annotate("text", y=515, x=1, label="+44%", size=5, hjust="left") +
  scale_y_continuous(limits=c(0, 600), breaks = seq(0, 500, by = 100)) +
  coord_flip() +
  theme_minimal_vgrid() +
  theme(legend.position="none", aspect.ratio = 1/3) +
  scale_fill_manual(values=c("#868686", "#0073C2", "#EFC000", "#CD534C")) +
  theme(axis.title=element_text(size=12),
        axis.text = element_text(size=12),
        plot.margin=unit(c(0,20,0,0),"mm"))
saveplot("numplayers.pdf", width=6, height=1.8, units="in")
```

# Bandwidth and Consistency

```{r}
df <- fread("bandwidth-consistency-experiment/results/opencraft-events.log", sep="\t")
df_pecosa <- fread("bandwidth-consistency-experiment/results/pecosa.log", sep="\t")

toHumanName <- function(v) {
  if (v == "policy-is") {
    "IS"
  } else if (v == "policy-isn") {
    "ISN"
  } else if (v == "policy-aoi") {
    "AOI"
  } else if (v == "policy-zero") {
    "ZERO"
  } else {
    "UNKOWN"
  }
}

# This plot can help detect any potential failures. The curves indicate the number of connected players, and should overlap for each policy.
df %>%
  filter(key == "numplayers") %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  ggplot(aes(x=timestamp, y=value, color=config)) +
  geom_line()

p1 <- df %>%
  filter(key == "mcpackets_sent") %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  filter(timestamp >= 60 & timestamp <= 290) %>%
  mutate(value = value - lag(value, default = NA)) %>%
  mutate(value = replace_na(value, 0)) %>%
  mutate(second = as.integer(timestamp)) %>%
  group_by(iteration, config, second) %>%
  summarize(value = sum(value)) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  ggplot(aes(x=config, y=value)) +
  geom_boxplot(aes(fill=fct_rev(config))) +
  stat_summary(fun.y=mean, geom="point", shape=21, size=2, color="black", fill="white") +
  scale_y_continuous(labels = function(x) floor(x/100000), limits=c(0, 1200000), breaks = seq(0, 1200000, by = 200000)) +
  scale_fill_manual(values = c("#868686", "#EFC000", "#CD534C")) +
  theme_half_open() +
  background_grid(major = "x") +
  labs(x="", y=bquote("\u00D7"*10^5~"messages per second"), fill="Dyconit policy") +
  coord_flip()

p2 <- df_pecosa %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  filter(timestamp >= 60 & timestamp <= 290) %>%
  mutate(net.packets_sent.ib0 = net.packets_sent.ib0 - lag(net.packets_sent.ib0, default = NA)) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  ggplot(aes(x=config, y=net.packets_sent.ib0)) +
  geom_boxplot(aes(fill=fct_rev(config))) +
  stat_summary(fun.y=mean, geom="point", shape=21, size=2, color="black", fill="white") +
  scale_y_continuous(labels = function(x) floor(x/1000), limits=c(0, 12000), breaks = seq(0, 12000, by = 2000)) +
  scale_fill_manual(values = c("#868686", "#EFC000", "#CD534C")) +
  theme_half_open() +
  background_grid(major = "x") +
  labs(x="", y=bquote("\u00D7"*10^3~"packets per second"), fill="Dyconit policy") +
  coord_flip()

p3 <- df_pecosa %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  filter(timestamp >= 60 & timestamp <= 290) %>%
  mutate(net.bytes_sent.ib0 = net.bytes_sent.ib0 - lag(net.bytes_sent.ib0, default = NA)) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  ggplot(aes(x=config, y=as.integer(net.bytes_sent.ib0))) +
  geom_boxplot(aes(fill=fct_rev(config))) +
  stat_summary(fun.y=mean, geom="point", shape=21, size=2, color="black", fill="white") +
  scale_y_continuous(labels = function(x) floor(x/1000000), limits=c(0, 12000000), breaks = seq(0, 12000000, by = 2000000)) +
  scale_fill_manual(values = c("#868686", "#EFC000", "#CD534C")) +
  theme_half_open() +
  background_grid(major = "x") +
  labs(x="", y=bquote("\u00D7"*10^6~"bytes per second"), fill="Dyconit policy") +
  coord_flip()

ggarrange(p1, p2, p3, ncol=1, nrow=3, common.legend = TRUE, legend="none", align="hv")
saveplot("messages-and-packets.pdf", width = 6, height = 4)

```

```{r}

p1 <- df %>%
  filter(key %in% c("dyconit_error_numerical")) %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  filter(timestamp >= 60 & timestamp <= 290) %>%
  mutate(value = value / 32) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  ggplot(aes(x=config, y=value, fill=fct_rev(config))) +
  geom_boxplot() +
  stat_summary(fun.y=mean, geom="point", shape=21, size=2, color="black", fill="white") +
  scale_y_continuous(labels = function(x) floor(x/10000), limits=c(0, 80000), breaks = seq(0, 80000, by = 20000)) +
  labs(x="", y=bquote("numerical error (\u00D7"*10^4~"in-game meters)"), fill="Dyconit policy") +
  scale_fill_manual(values = c("#868686", "#EFC000", "#CD534C")) +
  theme_half_open() +
  background_grid(major = "x") +
  theme(legend.position = "none") +
  guides(fill = guide_legend(reverse = TRUE)) +
  coord_flip()

px <- df %>%
  filter(key %in% c("dyconit_error_staleness")) %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  filter(timestamp >= 60 & timestamp <= 290) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  ggplot(aes(x=config, y=value, fill=fct_rev(config))) +
  geom_boxplot() +
  stat_summary(fun.y=mean, geom="point", shape=21, size=2, color="black", fill="white") +
  scale_y_continuous(labels = function(x) floor(x/100000), limits=c(0, 1600000), breaks = seq(0, 1600000, by = 400000)) +
  labs(x="", y=bquote("staleness (\u00D7"*10^5~"ms)"), fill="Dyconit policy") +
  scale_fill_manual(values = c("#868686", "#EFC000", "#CD534C")) +
  theme_half_open() +
  background_grid(major = "x") +
  theme(legend.position = "none") +
  guides(fill = guide_legend(reverse = TRUE)) +
  coord_flip()

p2 <- df %>%
  filter(key %in% c("tick")) %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  filter(timestamp >= 60 & timestamp <= 290) %>%
  mutate(config = map_chr(config, toHumanName)) %>%
  ggplot(aes(x=config, y=value, fill=fct_rev(config))) +
  geom_boxplot() +
  stat_summary(fun.y=mean, geom="point", shape=21, size=2, color="black", fill="white") +
  ylim(0, 100) +
  scale_fill_manual(values = c("#868686", "#EFC000", "#CD534C")) +
  theme_half_open() +
  theme(legend.position = "none") +
  background_grid(major = "x") +
  labs(x="", y="tick duration (ms)", fill="Dyconit policy") +
  coord_flip()

ggarrange(p1, px, p2, ncol=1, nrow=3, common.legend = TRUE, legend="none", align="v")
saveplot("error-vs-performance.pdf", width = 6, heigh = 4)
```

# Dynamic Consistency

```{r}
df <- fread("dynamic-consistency-experiment/results/opencraft-events.log", sep="\t")

numplayers <- df %>%
  filter(key == "numplayers") %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  ggplot(aes(x=timestamp, y=as.double(value))) +
  geom_line() +
  theme_half_open() +
  background_grid(major="y") +
  annotate("rect", xmin=-Inf, xmax=50, ymin=-Inf, ymax=Inf, fill="gray", alpha=0.4) +
  annotate("rect", xmin=165, xmax=225, ymin=-Inf, ymax=Inf, fill="red", alpha=0.3) +
  xlim(0, 360)
numplayers

strikethrough <- function(x) str_replace_all(x, "(?<=.)", "\u0336")
zero <- expression(bold("ZERO")*"/IS")
is <- expression("ZERO/"*bold("IS"))

tick <- df %>%
  filter(key == "tick") %>%
  group_by(iteration, config) %>%
  mutate(timestamp = (timestamp - min(timestamp)) / 1000) %>%
  mutate(rollmeanvalue = rollmean(as.double(value), 100, fill = NA)) %>%
  mutate(config = ifelse(config == "policy-zero", "ZERO", "ZERO/IS")) %>%
  mutate(markers = ifelse((row_number() + 5000) %% 1000 == 0, rollmeanvalue, NA)) %>%
  ggplot(aes(x=timestamp, y=rollmeanvalue, color=config)) +
  annotate("rect", xmin=-Inf, xmax=Inf, ymin=50, ymax=Inf, fill="red", alpha=0.3) +
  # Uncomment lines below to add (hardcoded) annotations.
  # annotate("rect", xmin=-Inf, xmax=60, ymin=-Inf, ymax=50, fill="gray", alpha=0.4) +
  # annotate("rect", xmin=189, xmax=241, ymin=-Inf, ymax=50, fill="orange", alpha=0.4) +
  # annotate("rect", xmin=180, xmax=195, ymin=-Inf, ymax=50, fill="gray", alpha=0.4) +
  # annotate("rect", xmin=195, xmax=240, ymin=-Inf, ymax=50, fill="orange", alpha=0.3) +
  # annotate("text", label="setup", x=45, y=3, angle=90, hjust = "left", size=5) +
  # annotate("text", label=zero, x=174, y=3, angle=90, hjust = "left", size=5) +
  # annotate("text", label=zero, x=256, y=3, angle=90, hjust = "left", size=5) +
  # annotate("text", label=is, x=215, y=3, angle=90, hjust = "left", size=5) +
  # annotate("text", label="300 players", x=165, y=0, angle=90, hjust = "left", size=5) +
  # annotate("text", label="setup", x=185, y=0, angle=90, hjust = "left", size=5) +
  # annotate("text", label="400 players", x=225, y=0, angle=90, hjust = "left", size=5) +
  # annotate("text", label="300 players", x=260, y=0, angle=90, hjust = "left", size=5) +
  annotate("text", x=5, y = 55, label = "Game overloaded", fontface = "bold", size = 5, hjust="left") +
  geom_hline(yintercept=50, linetype="dashed", color = "red", size=1) +
  geom_point(aes(y=markers, shape=config), size=3) +
  geom_line(aes(linetype=config)) +
  xlim(0, 360) +
  labs(x="time (s)", y="tick duration (ms)", color="Dyconit policy", shape="Dyconit policy", linetype="Dyconit policy") +
  theme_half_open() +
  background_grid(major="y") +
  theme(legend.justification=c(1,0), legend.position=c(1,0.015), legend.background = element_rect(fill="white")) +
  scale_color_manual(values = c("#868686", "#79B501"))
tick
saveplot("robustness.pdf", width=6, height=3)
```