Performance Overview of SOM Implementations
===========================================

```{r echo=FALSE, message=FALSE, warning=FALSE, results='hide', errors=FALSE}
# load libraries, the data, and prepare it
if (Sys.getenv("RSTUDIO") == "1") { setwd("/Users/smarr/Projects/PostDoc/SELF-OPT-INT/performance-overview") }

source("../scripts/libraries.R", chdir=TRUE)
data <- load_data_file("perf-overview.data")
data <- subset(data, select = c(Value, Unit, Benchmark, VM, Suite))
data <- prepare_vm_names(data)
steady <- droplevels(subset(data, grepl("som", Suite) | grepl("java", Suite) | grepl("startup", Suite)))
```


Overall Performance
-------------------

As a first, and misleading impression, an aggregated overview over the results
in form of a simple bar chart:

**TODO** add proper error bars, and read up on effect size confidence intervals.

```{r echo=FALSE, fig.width=6, fig.height=4, dev='svg'}
# aggregate results for display
stats <- ddply(steady, ~ Benchmark + VM + Suite,
               summarise,
               Time.mean                 = mean(Value),
               Time.geomean              = geometric.mean(Value),
               Time.stddev               = sd(Value),
               Time.median               = median(Value),
               max = max(Value),
               min = min(Value))

# normalize for each benchmark separately to the Java baseline
norm <- ddply(stats, ~ Benchmark, transform,
              RuntimeRatio = Time.mean / Time.mean[VM == "Java8"])

# summarize to VMs
vms <- ddply(norm, ~ VM,
             summarise,
             RunRatio.geomean = geometric.mean(RuntimeRatio))
```

```{r som-box-overview, echo=FALSE, fig.width=5, fig.height=2.5, dev='svg'}
 soms <- droplevels(subset(norm, VM == "Java8" | VM == "RPySOM" | VM == "RTruffleSOM" | VM == "SOM++-interp" | VM == "TruffleSOM" | VM == "SOM-interp"))
 name_map <-     list("Java8"                 = "Java",
                      "SOM++-interp"          = "SOM++",
                      "SOM-interp"            = "SOM")
 # Rename
 levels(soms$VM)  <- map_names(levels(soms$VM), name_map)


 plot <- ggplot(soms, aes(x=reorder(VM, RuntimeRatio, FUN=function (x) -median(x)), VM, y=RuntimeRatio)) +
  geom_hline(aes(yintercept=1), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=10), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=100), colour="#cccccc", linetype="dashed") +
  geom_hline(aes(yintercept=1000), colour="#cccccc", linetype="dashed") +
  geom_boxplot(fill=get_color(5, 7)) + theme_bw() + theme_simple() + theme(axis.text.x = element_text(angle= 90, vjust=0.5, hjust=1)) +
  scale_y_log10(breaks=c(1,2,3,10,20,30,50,100,200,300,500,1000)) + #limit=c(0,30), breaks=seq(0,100,5), expand = c(0,0)
  coord_flip() + xlab("Runtime, normalized\nto Java (lower is better)")
 plot
```

```{r echo=FALSE, fig.width=7, fig.height=5, dev='svg'}
# create a simple bar chart
# plot <- ggplot(norm, aes_string(x="VM", y="RuntimeRatio"))
# plot <- plot + geom_bar(stat="identity",
#                    colour=get_color(5, 6),
#                    size=.3,        # Thinner lines
#                    fill=get_color(5, 7),
#                    width=0.75) +
#   scale_y_continuous(limit=c(0,30),
#                                                     breaks=seq(0,100,5),
#                                                     expand = c(0,0)) +
#   ylab("Runtime, normalized to Java (lower is better)")
# plot <- plot + facet_wrap(~ Benchmark)
# plot <- plot +
#     theme_bw() + theme_simple() +
#     theme(axis.text.x          = element_text(angle= 90, vjust=0.5, hjust=1))
#   plot

```

```{r echo=FALSE, results='asis'}
# writeLines("<div class='full center'>")
# t <- tabular(Justify(l,data=l)*VM ~ Format(digits=2)*Heading('Runtime factor compared to Java8')*Benchmark*Heading()*RuntimeRatio*Justify(data=r)*(Heading()*mean),  data=droplevels(norm))
# html(t)
# writeLines("</div>")
```

```{r echo=FALSE, results='asis'}
# writeLines("<div class='full center'>")
# t <- tabular(Justify(l,data=l)*VM ~ Format(digits=2)*Heading('Runtime in ms')*Benchmark*Heading()*Value*Justify(data=r)*(mean + sd),  data=droplevels(steady))
# html(t)
# writeLines("</div>")
```




TODO
====

**TODO:**
 - Language Implementations:
    - LuaJIT 2
    - SOM (SOM++, CSOM, SOM (java))

 - some info on Benchmarks:
    - Mandelbrot (represents numerical performance)
    - Delta Blue (represents OO programs)
    - Richards   (represents OO programs)

