Performance Overview
====================

In order to give a brief intuition of the current state of self-optimizing
interpreters compared to other language implementations, we briefly compare 
the performance of a set of classic benchmarks for Java, C, JRuby, and the SOM
implementations.

Note, for the moment, the benchmark set is not yet complete for all languages.

```{r echo=FALSE, message=FALSE, warning=FALSE, results='hide', errors=FALSE}
# load libraries, the data, and prepare it
if (Sys.getenv("RSTUDIO") == "1") { setwd("/Users/smarr/Projects/PostDoc/SELF-OPT-INT/performance-overview") }

options(scipen = 5)

source("../scripts/libraries.R", chdir=TRUE)
data <- load_data_file("perf-overview.data")
data <- subset(data, select = c(Value, Unit, Benchmark, VM, Suite))
data <- prepare_vm_names(data)
```


Overall Performance
-------------------

As a first, and misleading impression, an aggregated overview over the results
in form of a simple bar chart:

**TODO** add proper error bars, and read up on effect size confidence intervals.

```{r echo=FALSE, fig.width=6, fig.height=4, dev='svg'}
# aggregate results for display
stats <- ddply(data, ~ Benchmark + VM + Suite,
               summarise,
               Time.mean                 = mean(Value),
               Time.geomean              = geometric.mean(Value),
               Time.stddev               = sd(Value),
               Time.median               = median(Value),
               max = max(Value),
               min = min(Value))

# normalize for each benchmark separately to the Java baseline
norm <- ddply(stats, ~ Benchmark, transform,
              RuntimeRatio = Time.mean / Time.mean[VM == "Java8"])

# summarize to VMs
vms <- ddply(norm, ~ VM,
             summarise,
             RunRatio.geomean = geometric.mean(RuntimeRatio))

compiled = droplevels(subset(vms, !grepl("-interp", VM)))
interp   = droplevels(subset(vms, grepl("-interp", VM)))

# create a simple bar chart
plot <- ggplot(compiled, aes_string(x="VM", y="RunRatio.geomean"))
plot <- plot + geom_bar(stat="identity",
                   colour=get_color(5, 6),
                   size=.3,        # Thinner lines
                   fill=get_color(5, 7),
                   width=0.75) +
    scale_y_continuous(breaks=seq(0,100,1), expand = c(0,0)) + # limit=c(0,11)
    ylab("Runtime, normalized to Java8\nlower is better") +
    theme_bw() + theme_simple() + theme(axis.text.x = element_text(angle= 90, vjust=0.5, hjust=1))
  plot

plot <- ggplot(interp, aes_string(x="VM", y="RunRatio.geomean"))
plot <- plot + geom_bar(stat="identity",
                   colour=get_color(5, 6),
                   size=.3,        # Thinner lines
                   fill=get_color(5, 7),
                   width=0.75) +
    scale_y_continuous(breaks=seq(0,500,25), expand = c(0,0)) + # limit=c(0,11)
    ylab("Runtime, normalized to Java8\nlower is better") +
    theme_bw() + theme_simple() + theme(axis.text.x = element_text(angle= 90, vjust=0.5, hjust=1))
  plot
```

```{r echo=FALSE, fig.width=6, fig.height=4, dev='svg'}
norm_compiled = droplevels(subset(norm, !grepl("-interp", VM)))
norm_interp   = droplevels(subset(norm, grepl("-interp", VM)))

 plot <- ggplot(norm_compiled, aes(x=VM, y=RuntimeRatio)) +
  geom_hline(aes(yintercept=1), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=10), colour="#cccccc", linetype="dashed") +
  geom_boxplot(fill=get_color(5, 7)) + theme_bw() + theme_simple() + theme(axis.text.x = element_text(angle= 90, vjust=0.5, hjust=1)) +
  scale_y_continuous(limit=c(0,30), breaks=seq(0,100,5), expand = c(0,0)) + 
  ylab("Runtime, normalized to Java8\nlower is better")
 plot

plot <- ggplot(norm_interp, aes(x=VM, y=RuntimeRatio)) +
  geom_hline(aes(yintercept=1), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=10), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=20), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=100), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=500), colour="#cccccc", linetype="dashed") +
  geom_boxplot(fill=get_color(5, 7)) + theme_bw() + theme_simple() + theme(axis.text.x = element_text(angle= 90, vjust=0.5, hjust=1)) +
  scale_y_log10(breaks=c(1, 2, 3, 5, 10, 15, 20, 30, 50, 100, 150, 200, 300, 500, 700, 900, 1000), expand = c(0,0))+
  #scale_y_continuous(breaks=seq(1,1000,25), expand = c(0,0)) + 
  ylab("Runtime, normalized to Java8\nlower is better")
 plot
```

```{r echo=FALSE, fig.width=7, fig.height=5, dev='svg'}
# create a simple bar chart
plot <- ggplot(norm_compiled, aes_string(x="VM", y="RuntimeRatio"))
plot <- plot + geom_bar(stat="identity",
                   colour=get_color(5, 6),
                   size=.3,        # Thinner lines
                   fill=get_color(5, 7),
                   width=0.75) +
  scale_y_continuous(limit=c(0,30),
                                                    breaks=seq(0,100,5),
                                                    expand = c(0,0)) +
  ylab("Runtime, normalized to Java (lower is better)")
plot <- plot + facet_wrap(~ Benchmark)
plot <- plot +
    theme_bw() + theme_simple() +
    theme(axis.text.x          = element_text(angle= 90, vjust=0.5, hjust=1))
  plot

plot <- ggplot(norm_interp, aes_string(x="VM", y="RuntimeRatio"))
plot <- plot + geom_bar(stat="identity",
                   colour=get_color(5, 6),
                   size=.3,        # Thinner lines
                   fill=get_color(5, 7),
                   width=0.75) +
  scale_y_continuous(breaks=seq(0,1000,100), expand = c(0,0)) +
  ylab("Runtime, normalized to Java (lower is better)")
plot <- plot + facet_wrap(~ Benchmark)
plot <- plot +
    theme_bw() + theme_simple() +
    theme(axis.text.x          = element_text(angle= 90, vjust=0.5, hjust=1))
  plot

```

```{r echo=FALSE, results='asis'}
writeLines("<div class='full center'>")
t <- tabular(Justify(l,data=l)*VM ~ Format(digits=2)*Heading('Runtime factor compared to Java8')*Benchmark*Heading()*RuntimeRatio*Justify(data=r)*(Heading()*mean),  data=droplevels(norm))
html(t)
writeLines("</div>")
```

```{r echo=FALSE, results='asis'}
writeLines("<div class='full center'>")
t <- tabular(Justify(l,data=l)*VM ~ Format(digits=1)*Heading('Runtime in ms')*Benchmark*Heading()*Value*Justify(data=r)*(mean + sd),  data=droplevels(data))
html(t)
writeLines("</div>")
```




TODO
====

**TODO:**
 - Language Implementations:
    - LuaJIT 2
    - SOM (SOM++, CSOM, SOM (java))

 - some info on Benchmarks:
    - Mandelbrot (represents numerical performance)
    - Delta Blue (represents OO programs)
    - Richards   (represents OO programs)

